RecursionError: maximum recursion depth exceeded while getting the str of an object - python

Was looking too so many examples here regarding this problem, but can not sort out for my example.
Any sugegstion will be appreciate, already having headdake witht his recursiona dn can not fix it out.
tree = {}
def populate_node(account):
node = '%(LOGIN)s,%(server_id)s' % account
tree[node]['login'] = account['LOGIN']
tree[node]['email'] = account['EMAIL'].lower()
tree[node]['server_id'] = account['server_id']
for account in accounts:
node = '%(LOGIN)s,%(server_id)s' % account
parent = None
if account['AGENT_ACCOUNT']:
parent = '%(AGENT_ACCOUNT)s,%(server_id)s' % account
if node not in tree:
tree[node] = {}
populate_node(account)
if parent:
tree[node]['parent'] = parent
if parent not in tree:
tree[parent] = {
'login': parent,
'server_id': account['server_id'],
'children': [node],
}
else:
if 'children' not in tree[parent]:
tree[parent]['children'] = [node]
else:
tree[parent]['children'].append(node)
def get_path(node, tree):
parent = node.get('parent')
node_login = str(str(node.get('login')) + ',' + str(node.get('server_id')))
if not parent:
return []
elif parent == node_login:
return [parent]
path = get_path(tree[parent], tree)
return [parent] + path
for k, v in tree.items():
v['path'] = get_path(v, tree)
v['level'] = len(v['path']) + (1 if v['login'] != v.get('parent') else 0)
By default :
tree = {}
node is an item of tree.
sample tree:
tree = {
'1987,mt4-demo-0': {
'login': 1987,
'email': 'email_1',
'server_id': 'mt4-demo-0'
},
'16044,mt4-demo-0': {
'login': 16044,
'email': 'email_2',
'server_id': 'mt4-demo-0'
},
'160877748,mt4-demo-0': {
'login': 160877748,
'email': 'email_3',
'server_id': 'mt4-demo-0'
}
}
And gettings everytme in this recursion error
RecursionError: maximum recursion depth exceeded while getting the str of an object

Your code assumes you are always dealing with an acyclic directed graph, but your input has at least one directed cycle in it, where one AGENT_ACCOUNT reference directly or indirectly points to another account that in turn has an AGENT_ACCOUNT value pointing back to the first account.
For example, if accounts is set to:
accounts = [
{'LOGIN': 'foo', 'EMAIL': 'foo#bar.com', 'server_id': 'server 1',
'AGENT_ACCOUNT': 'bar'},
{'LOGIN': 'bar', 'EMAIL': 'bar#bar.com', 'server_id': 'server 1',
'AGENT_ACCOUNT': 'foo'}]
then tree becomes:
{'bar,server 1': {'children': ['foo,server 1'],
'email': 'bar#bar.com',
'login': 'bar',
'parent': 'foo,server 1',
'server_id': 'server 1'},
'foo,server 1': {'children': ['bar,server 1'],
'email': 'foo#bar.com',
'login': 'foo',
'parent': 'bar,server 1',
'server_id': 'server 1'}}
Note that foo has an AGENT_ACCOUNT that points to bar, and bar points back to foo, forming a cycle.
This then will produce an infinite recursion error on either of those two entries:
>>> get_path(tree['bar,server 1'], tree)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 8, in get_path
File "<stdin>", line 8, in get_path
File "<stdin>", line 8, in get_path
[Previous line repeated 994 more times]
File "<stdin>", line 2, in get_path
RecursionError: maximum recursion depth exceeded while calling a Python object
You can detect such cycles early and exit with a clearer error message:
def get_path(node, tree, seen=None):
if seen is None:
seen = set()
parent = node.get('parent')
if parent:
if parent in seen:
raise ValueError(
'Already handled {!r}, cycle detected. '
'Check all of {}'.format(
parent, sorted(seen)))
seen.add(parent)
node_login = '{0[login]},{0[server_id]}'.format(node) # cleaner method to generate the key
if not parent:
return []
elif parent == node_login:
return [parent]
path = get_path(tree[parent], tree, seen) # pass seen along to recursive calls
return [parent] + path
Running this updated version on the same tree now produces:
>>> get_path(tree['bar,server 1'], tree)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 16, in get_path
File "<stdin>", line 16, in get_path
File "<stdin>", line 9, in get_path
ValueError: Already handled 'foo,server 1', cycle detected. Check all of ['bar,server 1', 'foo,server 1']fted by one space
I assumed that such cycles are errors. If they are not, simply return the path up to that point with if parent in seen: return [] (so ignore the cycle), but you'll have a version of the path for every member of the cycle, each path a rotated version of the next.
You should really fix your account information, and remove such cycles. If you need to find all such cycles, you could use:
from collections import deque
def find_all_cycles(tree):
visited, cycles, path = set(), [], []
queue = deque(sorted(tree))
while queue:
key = queue.pop()
if key in visited:
continue
visited.add(key)
path.append(key)
parent = tree[key].get('parent')
if not parent:
path = []
elif parent in visited:
# cycle detected!
cycles.append(path + [parent])
path = []
else:
queue.append(parent)
return cycles

Related

ValueError when accessing a Table after inserting a ROW object to list:reference field

I am getting a ValueError for the DAL query db(db.posts.id == request.vars["post"]).select().first(), with the following TraceBack:
Traceback (most recent call last):
File "E:\Programming\Python\web2py\gluon\restricted.py", line 204, in restricted
exec ccode in environment
File "E:/Programming/Python/web2py/applications/vote_up/controllers/default.py", line 109, in <module>
File "E:\Programming\Python\web2py\gluon\globals.py", line 172, in <lambda>
self._caller = lambda f: f()
File "E:/Programming/Python/web2py/applications/vote_up/controllers/default.py", line 67, in vote
if request.vars["post"]: var = db(db.posts.id == request.vars["post"]).select().first()
File "E:\Programming\Python\web2py\gluon\dal.py", line 6333, in select
return self.db._adapter.select(self.query,fields,attributes)
File "E:\Programming\Python\web2py\gluon\dal.py", line 1287, in select
return self.parse(rows,self._colnames)
File "E:\Programming\Python\web2py\gluon\dal.py", line 1533, in parse
colset[fieldname] = bar_decode_integer(value)
File "E:\Programming\Python\web2py\gluon\dal.py", line 4408, in bar_decode_integer
return [int(x) for x in value.split('|') if x.strip()]
ValueError: invalid literal for int() with base 10: "<Row {'joined': datetime.datetime(2012, 2, 26, 17, 42, 26), 'password': 'SS', 'id': 1, 'name': 'a', 'reputation': 0}>"
This happened after I changed my controller to:
def vote():
'''Add a vote'''
path_to=session.path_to
if request.vars["post"]: var = db(db.posts.id == request.vars["post"]).select().first()
elif request.vars["answer"]: var = db(db.answers.id == request.vars["answer"]).select().first()
elif request.vars["comment"]: var = db(db.comments.id == request.vars["comment"]).select().first()
elif request.vars["comment_r"]: var = db(db.comments_r.id == request.vars["comment_r"]).select().first()
else: return None
inc = 0
if int(request.vars["up"])==1:
inc=1
if not var.v_up: var.v_up = []
if session.user in var.v_up: return var.votes
var.update_record(
votes=var.votes+inc,
v_up = var.v_up+[session.user]
)
elif int(request.vars["up"])==0:
inc=-1
if not var.v_dn: var.v_dn = []
if session.user in var.v_dn: return var.votes
var.update_record(
votes=var.votes+inc,
v_dn = var.v_dn+[session.user]
)
return var.votes
The session.user is a ROW object of the users table, like:
<Row {'joined': datetime.datetime(2012, 2, 26, 17, 42, 26), 'password': 'SS', 'id': 1, 'name': 'a', 'reputation': 0}>
The tables are structured like:
db.define_table('posts',
db.Field('title', 'string'),
db.Field('message', 'text'),
db.Field('user', 'reference users'),
db.Field('votes', 'integer', default=0),
db.Field('v_up', 'list:reference users'),
db.Field('v_dn', 'list:reference users'),
db.Field('time', 'datetime', default=datetime.utcnow())
)
db.define_table('users',
db.Field('name', 'string'),
db.Field('password', 'password'),
db.Field('reputation', 'integer', default=0),
db.Field('joined', 'datetime', default=datetime.utcnow())
)
Am I missing something trivial again?
UPDATE:
Looks like the posts table is corrupted since I get the same ValueError even for print db(db.posts).select(). Could v_up = var.v_up+[session.user] have corrupted the table? Why? And is there someway to recover it?
UPDATE:
Confirmed that v_up = var.v_up+[session.user] is corrupting the table. What is the right way to use a list:refernce field?
The list:refernce Field in DAL is a glorified integer-list field that holds ids of rows of the referenced table.
Therefore when you insert a ROW object into that field, it causes a ValueError as a ROW object isn't an integer id from the table.
To fix this, replace the session.user ROW object with its integer-id, session.user.id.
Here is an example from the Web2Py book:
>>> db.define_table('tag',Field('name'),format='%(name)s')
>>> db.define_table('product',
Field('name'),
Field('tags','list:reference tag'))
>>> a = db.tag.insert(name='red')
>>> b = db.tag.insert(name='green')
>>> c = db.tag.insert(name='blue')
>>> db.product.insert(name='Toy Car',tags=[a, b, c])
>>> products = db(db.product.tags.contains(b)).select()
>>> for item in products:
print item.name, item.tags
Toy Car [1, 2, 3]
>>> for item in products:
print item.name, db.product.tags.represent(item.tags)
Toy Car red, green, blue
Note that the insert method returns the id of the ROW inserted.

TypeError in dictionary

I am trying to pull specific values from a dictionary but my program keeps throwing a TypeError on some values from the dictionary. Can someone tell me why?
python program
import csv,os,re,sys
input_dict = csv.DictReader(open("./MCPlayerData/AllPlayerData2.csv"))
def list_scale_values(regexmatch,stopOnNA,person):
data=[]
for key in person:
if re.match(regexmatch, key):
try:
data.append(int(person[key]))
except (ValueError):
data.append('NA')
if(stopOnNA):
data=['NA'] #if any NA return NA
break
return data
try:
for person in input_dict:
print(list_scale_values(r'npi[0-9]+',True,person))
except (TypeError):
type, value, traceback = sys.exc_info()
print(type)
print(value)
print(traceback)
print '\n---\n',person,'\n---'
sys.exit()
print('DONE')
error output
Traceback (most recent call last):
File "correlations.py", line 22, in <module>
print(list_scale_values(r'npi[0-9]+',True,person))
File "correlations.py", line 9, in list_scale_values
if re.match(regexmatch, key):
File "/usr/lib/python2.7/re.py", line 137, in match
return _compile(pattern, flags).match(string)
TypeError: expected string or buffer
To avoid this problem check if the key is a valid string or buffer before a regular expression is made.
so instead of if re.match(regexmatch, key):
write the following code
if key != None and re.match(regexmatch, key):
To answer the question why it would happen for some person in the csv dictionary?
If the csv file does not take into consideration for all the columns in a row for each row this situation is totally possible for only certain rows, in this case people
To give an example :
consider the following CSV file
name,type,address,value
zoo,1,111,2
foo,2,222,,
bar,3,333,5
This would get the following results for the persons in csv.DictReader(open("filename.csv"))
{'type': '1', 'name': 'abc', 'value': '2', 'address': '111'}
{None: [''], 'type': '2', 'name': 'foo', 'value': '', 'address': '222'}
{'type': '3', 'name': 'bar', 'value': '5', 'address': '333'}
So in this case it would work for people[0] and people[2] but fail at people[1]

JSON sub for loop produces KeyError, but key exists

I'm trying to add the JSON output below into a dictionary, to be saved into a SQL database.
{'Parkirisca': [
{
'ID_Parkirisca': 2,
'zasedenost': {
'Cas': '2016-10-08 13:17:00',
'Cas_timestamp': 1475925420,
'ID_ParkiriscaNC': 9,
'P_kratkotrajniki': 350
}
}
]}
I am currently using the following code to add the value to a dictionary:
import scraperwiki
import json
import requests
import datetime
import time
from pprint import pprint
html = requests.get("http://opendata.si/promet/parkirisca/lpt/")
data = json.loads(html.text)
for carpark in data['Parkirisca']:
zas = carpark['zasedenost']
free_spaces = zas.get('P_kratkotrajniki')
last_updated = zas.get('Cas_timestamp')
parking_type = carpark.get('ID_Parkirisca')
if parking_type == "Avtomatizirano":
is_automatic = "Yes"
else:
is_automatic = "No"
scraped = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
savetodb = {
'scraped': scraped,
'id': carpark.get("ID_Parkirisca"),
'total_spaces': carpark.get("St_mest"),
'free_spaces': free_spaces,
'last_updated': last_updated,
'is_automatic': is_automatic,
'lon': carpark.get("KoordinataX_wgs"),
'lat': carpark.get("KoordinataY_wgs")
}
unique_keys = ['id']
pprint savetodb
However when I run this, it gets stuck at for zas in carpark["zasedenost"] and outputs the following error:
Traceback (most recent call last):
File "./code/scraper", line 17, in <module>
for zas in carpark["zasedenost"]:
KeyError: 'zasedenost'
I've been led to believe that zas is in fact now a string, rather than a dictionary, but I'm new to Python and JSON, so don't know what to search for to get a solution. I've also searched here on Stack Overflow for KeyErrror when key exist questions, but they didn't help, and I believe that this might be due to the fact that's a sub for loop.
Update: Now, when I swapped the double quotes for single quotes, I get the following error:
Traceback (most recent call last):
File "./code/scraper", line 17, in <module>
free_spaces = zas.get('P_kratkotrajniki')
AttributeError: 'unicode' object has no attribute 'get'
I fixed up your code:
Added required imports.
Fixed the pprint savetodb line which isn't valid Python.
Didn't try to iterate over carpark['zasedenost'].
I then added another pprint statement in the for loop to see what's in carpark when the KeyError occurs. From there, the error is clear. (Not all the elements in the array in your JSON contain the 'zasedenost' key.)
Here's the code I used:
import datetime
import json
from pprint import pprint
import time
import requests
html = requests.get("http://opendata.si/promet/parkirisca/lpt/")
data = json.loads(html.text)
for carpark in data['Parkirisca']:
pprint(carpark)
zas = carpark['zasedenost']
free_spaces = zas.get('P_kratkotrajniki')
last_updated = zas.get('Cas_timestamp')
parking_type = carpark.get('ID_Parkirisca')
if parking_type == "Avtomatizirano":
is_automatic = "Yes"
else:
is_automatic = "No"
scraped = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
savetodb = {
'scraped': scraped,
'id': carpark.get("ID_Parkirisca"),
'total_spaces': carpark.get("St_mest"),
'free_spaces': free_spaces,
'last_updated': last_updated,
'is_automatic': is_automatic,
'lon': carpark.get("KoordinataX_wgs"),
'lat': carpark.get("KoordinataY_wgs")
}
unique_keys = ['id']
pprint(savetodb)
And here's the output on the iteration where the KeyError occurs:
{u'A_St_Mest': None,
u'Cena_dan_Eur': None,
u'Cena_mesecna_Eur': None,
u'Cena_splosno': None,
u'Cena_ura_Eur': None,
u'ID_Parkirisca': 7,
u'ID_ParkiriscaNC': 72,
u'Ime': u'P+R Studenec',
u'Invalidi_St_mest': 9,
u'KoordinataX': 466947,
u'KoordinataX_wgs': 14.567929171694901,
u'KoordinataY': 101247,
u'KoordinataY_wgs': 46.05457609543313,
u'Opis': u'2,40 \u20ac /dan',
u'St_mest': 187,
u'Tip_parkirisca': None,
u'U_delovnik': u'24 ur (ponedeljek - petek)',
u'U_sobota': None,
u'U_splosno': None,
u'Upravljalec': u'JP LPT d.o.o.'}
Traceback (most recent call last):
File "test.py", line 14, in <module>
zas = carpark['zasedenost']
KeyError: 'zasedenost'
As you can see, the error is quite accurate. There's no key 'zasedenost' in the dictionary. If you look through your JSON, you'll see that's true for a number of the elements in that array.
I'd suggest a fix, but I don't know what you want to do in the case where this dictionary key is absent. Perhaps you want something like this:
zas = carpark.get('zasedenost')
if zas is not None:
free_spaces = zas.get('P_kratkotrajniki')
last_updated = zas.get('Cas_timestamp')
else:
free_spaces = None
last_updated = None

Python Multiprocessing: Children sending variable back to parent

I have a pool that runs a function on a list of IPs and returns an integer (0 through 99). The number of children is determined by the number of IPs. The idea is the parent builds a dict with IP keys and the integers values.
I need the children to report the number back to the parent process so it can save it in a pickled file. Assigning the integer variable as global has not worked.
Reading a pickled file in as a dict:
infile = open(server_file, 'rb')
servers = pickle.load(infile)
infile.close()
Result:
servers = {'10.116.35.141': ''}
The function:
def check(server):
response = urllib2.urlopen("http://"+server+"/avicapture.html")
tall = response.read() # puts the data into a string
html = tall.rstrip()
match = re.search('.*In Progress \((.*)%\).*', html)
if match:
global temp
global results
temp = match.group(1)
results = temp
servers[server] = temp # not setting key to temp in parent process like I want.
The pool:
p = Pool(len(servers))
p.map(check, servers)
p.close()
p.join()
My desired result in the parent process:
servers = {'10.116.35.141': 'integer from child'}
dano:
It is working now. Sort of. Here's the current state:
Starting with a dict of:
servers = { '10.116.35.141': '' }
def check(server):
response = urllib2.urlopen("http://"+server+"/avicapture.html")
tall = response.read() # puts the data into a string
html = tall.rstrip()
match = re.search('.*In Progress \((.*)%\).*', html)
global temp
if match:
global temp
global results
temp = match.group(1)
results = temp
servers[server] = temp
print servers
return str(temp)
print("Results: " +results)
Here is the pooling:
if __name__ == "__main__":
print "Print Servers:\n"+str(servers)
print "Starting pool"
p = Pool(len(servers))
print "p.map(check, servers)"
results = p.map(check, servers)
p.close()
p.join()
d = dict(ent for ent in zip(servers, results))
print "d = dict(ent for ent in zip(servers, results))\nResults in.."
print(d)
servers = d
print "Translation into servers dict:"
print(servers)
This results in the desired:
Print Servers:
{'10.116.35.141': ''}
Starting pool
p.map(check, servers)
{'10.116.35.141': '96'}
Server: 10.116.35.141 ... 96%
d = dict(ent for ent in zip(servers, results))
Results in..
{'10.116.35.141': '96'}
Translation into servers dict:
{'10.116.35.141': '96'}
The problem starts when I have a dict with more than one key. For instance:
servers = { '10.116.35.141': '', '10.116.35.1': '' }
The way the additional keys are added is:
input = raw_input("Add IP: ")
if input in servers:
print "Server already in servers list."
elif input == "":
print "No input."
else:
global servers
servers[input] = ""
I get StringIO errors:
Print Servers:
{'192.168.91.236': '', '10.116.35.141': ''}
Starting pool
p.map(check, servers)
Process PoolWorker-1:
Traceback (most recent call last):
File "/usr/lib64/python2.6/multiprocessing/process.py", line 232, in _bootstrap
self.run()
File "/usr/lib64/python2.6/multiprocessing/process.py", line 88, in run
self._target(*self._args, **self._kwargs)
File "/usr/lib64/python2.6/multiprocessing/pool.py", line 71, in worker
put((job, i, result))
File "/usr/lib64/python2.6/multiprocessing/queues.py", line 366, in put
return send(obj)
PicklingError: Can't pickle <type 'cStringIO.StringO'>: attribute lookup cStringIO.StringO failed
{'192.168.91.236': '', '10.116.35.141': '1'}
Traceback (most recent call last):
File "./bsdae", line 233, in <module>
results = p.map(check, servers)
File "/usr/lib64/python2.6/multiprocessing/pool.py", line 148, in map
return self.map_async(func, iterable, chunksize).get()
File "/usr/lib64/python2.6/multiprocessing/pool.py", line 422, in get
raise self._value
UnboundLocalError: local variable 'message' referenced before assignment
pool.map actually allows you to return the results directly:
def check(val):
return val + 5
if __name__ == "__main__":
servers = [1,3,4,5]
p = Pool(len(servers))
results = p.map(check, servers)
p.close()
p.join()
print(results)
Output:
[6, 8, 9, 10]
You can make your check function return the result, and it will end up in a list returned by the map call.
So if you have:
servers = ['1.1.1.1', '2.2.2.2', '3.3.3.3']
results = p.map(check, servers) # Let's say this returns [1,2,3]
You can get the dict you want like this:
d = dict(ent for ent in zip(servers, results))
print(d)
Output:
{'3.3.3.3': 3, '1.1.1.1': 1, '2.2.2.2': 2}

Dendropy leaf_iter() error with lambda function

I'm doing some phylogenetics in Dendropy:
I'm trying, following the tutorial, to traverse all the leaves of a tree using a filter (a lambda function).
Using the lambda function gives an error if the iterator is leaf_iter, but not if the iterator is any other one (for example postorder_node_iterate). What am I missing?
The traceback is some hundreds line long so I just show you an extract:
Traceback (most recent call last):
File "<pyshell#274>", line 1, in <module>
for node in tree_test.leaf_iter(survival):
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 915, in leaf_iter
for node in self.seed_node.leaf_iter(filter_fn=filter_fn):
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 2444, in leaf_iter
for node in self.postorder_iter(filter_fn):
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 2427, in postorder_iter
if filter_fn is None or filter_fn(node):
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 2441, in <lambda>
filter_fn = lambda x: x.is_leaf() and filter_fn(x) or None
[... line 2441 ...]
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 2441, in <lambda>
filter_fn = lambda x: x.is_leaf() and filter_fn(x) or None
File "build\bdist.win32\egg\dendropy\dataobject\tree.py", line 2537, in is_leaf
return bool(not self._child_nodes)
RuntimeError: maximum recursion depth exceeded while calling a Python object
To test the problem you can use the following code, pretty much taken from the tutorial:
#call a bit of modules and build a simulated tree
import dendropy
import random
from dendropy import TaxonSet, Tree, TreeList, treesim
taxa = dendropy.TaxonSet(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'])
tree_test = treesim.birth_death(0.4, 0.1, ntax=8, taxon_set=taxa, repeat_until_success=True)
#define an evolutionary character
def evolve_char(tree, start=1.0):
for node in tree.preorder_node_iter():
if node.parent_node is None:
node.value = 1.0
else:
node.value = random.gauss(node.parent_node.value, node.edge.length)
return tree
#evolve the character on the simulated tree
evolve_char(tree_test)
#positive_char select the nodes with positive character
positive_char = lambda node: True if node.value > 0. else False
#iter with postorder_node_iter (no problem)
for node in tree_test.postorder_node_iter():
print("Node %s with value %s" % (node.taxon, node.value))
for node in tree_test.leaf_iter(positive_char):
print("Node %s with value %s" % (node.taxon, node.value))
#iter with leaf_iter without filtering (no problem)
for node in tree_test.leaf_iter():
print("Node %s with value %s" % (node.taxon, node.value))
#iter with leaf_iter and filtering (problem!)
for node in tree_test.leaf_iter(positive_char):
print("Node %s with value %s" % (node.taxon, node.value))
Ok, that was a bug. Just fixed in the new github release.
I'll leave this here as somebody else may find the same problem.

Resources