import os, sys, types, logging, re, hashlib, io
from collections import OrderedDict
if __name__ == "__main__":
sys.path.append(".")
from simtoolkit.tree import tree
from simtoolkit.database import db
else:
from .tree import tree
from .database import db
from numpy import *
from functools import reduce
class methods:
"""
The class `methods` is a container for all parameters and (maybe) some results of a simulation.
This class seems as a tree of objects, so parameter can be called by name.
EXAMPLE: methods["/parameter/a"] or methods["parameter"]["a"]
The method class parses configuration file(s) and command lines arguments to create a
text tree methods_txt.
The text tree contains parameter:string_representation_of_the_value and is used for holding
textual values of parameters before the generation of python objects.
To generate a python object for a given name read the parameter value, or call
generate(parameter_name), or call generate() without parameters for building all parameters.
NOTE generate(parameter_name) will generate all dependencies required to make the requested parameter.
If, for example, /p=@/k@*2 and /k=exp(@/x@) and /x=pi, both /x and /k would be generated by call
generate("/p")
"""
is_lambda = lambda self, value : isinstance(value, types.LambdaType) and value.__name__ == '<lambda>'
__check_pattern__ = lambda self, key, pat : key[ :len(pat)] == pat
__kcehc_pattern__ = lambda self, key, pat : key[-len(pat):] == pat
def __init__(self,
default_conf, target, localcontext,
argvs=None,
# dbsymbol = "db=", dbsepcartor = ":",
# cfsymbol = "conf=", cfseparator = ":",
hsymbol = "/", vseparator = "=",
refsymbol = "@", strefsymbol = "$", refhash = "#",
mlvsymbol = "\\", mseparator = ";",
groupopen = "{", groupend="}",
isymbol="`",
mmopen ="/{", mmend="}",
pure=False):
"""
creates the object of methods. __init__ needs 3 parameters and accepts lots of options
@param default_conf- a string with default configuration, which methods parses to extract
default parameter set (maybe an open text file or
a list of string and/or open files).
@param target - the name of methods object which will be created by this constructor
@param localcontext- local namespace where all exec(s) should be run.
can be alter by localcontext parameter in generate function
if None, methods uses current context from globals()
---
@opt argvs - command line arguments, which will be processed after defaults.
old interface -->
@opt dbsymbol - (a set of) symbol(s) to indicate that some parameters should be read
from data base (db= by default)
@opt dbsepcartor - a separator to segregate database name/url, record hash/time-stamp and
parameter name(s) (: by default)
EXAMPLES:
url access, extracts record using time-stamp:
db=mysql:myneuron.org:2016-03-02/*:/Populations/E:/Population/P:/Connections
local STKDB file access, extracts record using hash:
db=simulations.stkdb:2a077d01a8a1018c6902b20b8bcfe1e90082b952:/Populations/AN:/Populations/SBC:/Populations/GBC
@opt cfsymbol - (a set of) symbol(s) to indicate that some parameters should be read
from additional configuration file (conf= by default)
@opt cfseparator - a separator to segregate configuration filename and
parameter name(s) (: by default)
EXAMPLES:
reads all parameters from an additional configuration file
conf=additional.conf
reads some parameters from an additional configuration file
conf=additional.conf:/Populations/AN:/Populations/SBC:/Populations/GBC
<---
@opt vseparator - a symbol to separate a parameter name from a value (= by default)
@opt hsymbol - a symbol to present hierarchical position within a parameter name (/ by default)
@opt refsymbol - a symbol to indicate a referenced name (@ by default)
@opt strefsymbol - a symbol to indicate a referenced name if the value should be converted back to a string ($ by default)
@opt refhash - a symbol to indicate a referenced name if the value should be a hash sum of the content (# by default)
@opt mlvsymbol - a symbol to indicate continuation of value line (\\ by default)
@opt mseparator - a symbol to separate a parameter=value couple and a message (; by default)
@opt groupopen - a symbol at the beginning of group subtree ({ by default)
@opt groupend - a symbol at the end of group subtree (} by default)
@opt isymbol - a symbol at the beginning and end of iterator in both a name and a value (` by default)
@opt mmopen - a (set of) symbol(s) at the beginning of main message, i.e. model title (/{ by default)
@opt mmend - a (set of) symbol(s) at the end of main message, i.e. model title (} by default)
@opt pure - if True parameters with wrong format will raise an exception (False by default).
"""
self.logger = logging.getLogger("simtoolkit.methods")
#--
self.dtarget, self.dlocalctx = target, localcontext
#---
# self.dbsymbol, self.dbsepcartor = dbsymbol, dbsepcartor
# self.cfsymbol, self.cfseparator = cfsymbol, cfseparator
#---
self.vseparator, self.hsymbol = vseparator, hsymbol
self.refsymbol, self.strefsymbol = refsymbol, strefsymbol
self.refhash, self.mseparator = refhash, mseparator
self.mlvsymbol = mlvsymbol
self.groupopen, self.groupend = groupopen, groupend
self.isymbol = isymbol
self.mmopen, self.mmend = mmopen, mmend
self.pure = pure
#---
self.methods = tree(hsymbol=self.hsymbol)
self.methods_txt = tree(hsymbol=self.hsymbol)
self.hashspace = tree(hsymbol=self.hsymbol)
self.iterate = OrderedDict()
self.stackcnt = 0
self.iterate_res = False
#---
self.mainmessage = ""
if type(default_conf) is str :
default_conf = [ io.StringIO(default_conf) ]
default_conf[0].name = "default"
elif type(default_conf) is tuple or type(default_conf) is list:
for cid,c in enumerate(default_conf):
if type(c) is not str and not isinstance(c, io.IOBase):
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __init__)")
self.logger.error(" element #{} of default_conf is not a string or file. {} is given".format(cid,type(c)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError(" element #{} of default_conf is not a string or file. {} is given".format(cid,type(c)))
default_conf = [ io.StringIO(c) if type(c) is str else c for c in default_conf ]
elif isinstance(default_conf, io.IOBase):
default_conf = [default_conf]
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __init__)")
self.logger.error(" default_conf should be a string or a file object or list/tuple of strings or file objects. {} is given".format(type(default_conf)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("default_conf should be a string or a file object or list/tuple of strings or file objects. {} is given".format(type(default_conf)))
self.__confreader__(default_conf)
#---
if argvs is None: return
if type(argvs) is str:
argvs = [ argvs ]
elif type(argvs) is tuple or type(argvs) is list:
if reduce(lambda x,y: x and type(y) is str, argvs, True): pass
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __init__)")
self.logger.error(" Not all arguments {} are strings".format(argvs))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Not all arguments {} are strings".format(argvs))
return
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __init__)")
self.logger.error(" Arguments should be a string or list/tuple of strings or file objects. {} is given".format(type(argvs)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Arguments should be a string or list/tuple of strings or file objects. {} is given".format(type(argvs)))
for arg in argvs:
parts = self.resolve_expression(arg)
if len(parts) != 2 :
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __init__)")
self.logger.error(" Cannot resolve argument {}".format(arg))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Cannot resolve argument {}".format(arg))
continue
arg,value = parts
if arg in self.methods_txt and not isinstance(self.methods_txt[arg], tree):
self.methods_txt[arg] = (value, self.methods_txt[arg][1] )
else :
self.methods_txt[arg] = (value,"--Command line argument--")
# Functions to read parameters from default configuration, stkdb and so on
def resolve_expression(self, expr, sep = None):
if sep is None: sep = self.vseparator
return [ x.strip(" \n\t\r") for x in expr.strip(" \n\t\r").split(sep,1) ]
def __confreader__(self, confile):
"""
reads configuration file(s) and creates a text tree.
it resolves groups and prepares iterators for further use
"""
def resolve_iterators(name, sep=self.isymbol):
copir = name.strip(" \n\t\r").split(sep)
if len(copir)%2: copir += [None]
result = ""
for prefix,var in zip(copir[::2], copir[1::2]):
if prefix is not None: result += prefix
if var is None: continue
parts = self.resolve_expression(var)
if len(parts) > 1:
arg,value = parts
#arg,value = arg,value
self.iterate[sep+arg+sep]=value
result += sep+parts[0]+sep
return result
if type(confile) is tuple or type(confile) is list:
for cfl in confile:
self.__confreader__(cfl)
return
message_on = False
groups = []
command, message, continue_on = "", "", False
#Step one read parameters tree from the file
for nl,line in enumerate(confile.readlines()):
l = line.strip("\n\r")
if len(l) == 0 : continue
if len(l.lstrip(" \t")) == 0 : continue
if l.lstrip(" \t")[0] == self.mseparator\
and not continue_on : continue
#Extracting the main message
if self.__check_pattern__(l,self.mmopen):
message_on = True
self.mainmessage += l[len(self.mmopen):]+"\n"
continue
elif message_on and self.__kcehc_pattern__(l,self.mmend):
self.mainmessage += l[:-len(self.mmend)]+"\n"
message_on = False
continue
elif message_on:
self.mainmessage += l+"\n"
continue
#Parsing the rest
l = line.strip(" \t")
vm = l.split(self.mseparator,1)
if len(vm) == 1:
vm = vm[0].strip(" \n\t\r"+self.mlvsymbol)
if self.__kcehc_pattern__(vm,self.groupopen):
#resolve ierators in a group
gname = resolve_iterators(vm[:-len(self.groupopen)].strip(" \n\t\r"+self.hsymbol))
groups.append(gname)
continue
elif self.__kcehc_pattern__(vm,self.groupend):
groups = groups[:-1]
continue
else:
self.logger.warning("----------------------------------------------------")
self.logger.warning(" METHODS ERROR in __confreader__)")
self.logger.warning(" Found line with no message divider {} at line {} of file {}: {}".format(self.mseparator, nl+1, confile.name,vm))
self.logger.warning("----------------------------------------------------")
if self.pure : raise ValueError(" Found line with no message divider {} at line {} of file {}".format(self.mseparator, nl+1, confile.name))
elif self.__kcehc_pattern__(vm[0].strip(" \n\t\r"),self.mlvsymbol):
command += vm[0].strip(" \n\t\r")[:-len(self.mlvsymbol)]
message += vm[1].replace("\n"," ")
if not continue_on: continue_on = nl+1
continue
else:
command += vm[0]
message += vm[1]
if self.__kcehc_pattern__(command.strip(" \n\t\r"),self.groupopen) or self.__kcehc_pattern__(command.strip(" \n\t\r"),self.groupend):
self.logger.warning("----------------------------------------------------")
self.logger.warning(" METHODS ERROR in __confreader__)")
self.logger.warning(" Found group beginning or end before a message at line {} of {}".format(nl+1, "DEFAULT" if "name" not in confile else confile.name))
self.logger.warning(" NOTE: it you are using dictionaries please convert them into a subtree(s) to avoid this message again.")
self.logger.warning("----------------------------------------------------")
try:
parts = self.resolve_expression( resolve_iterators(command) )+[message]
parts = "/".join([""]+groups+[""])+parts[0].strip(" \n\t\r/"), parts[1].strip(" \n\t\r"),parts[2].strip(" \n\t\r")
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __confreader__)")
self.logger.error(" Cannot parse command \'{}\' or message \'{}\' ".format(command,message))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot parse command \'{}\' or message \'{}\' ".format(command,message))
self.methods_txt[parts[0]]=parts[1],parts[2]
command, message, continue_on = "", "", False
confile.close()
def __read_conf__(self,conf,parameters=None):
try:
xconf = methods(conf,"xconf",locals())
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_conf__)")
self.logger.error(" Cannot read configuration file {}: {}".format(conf,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot read configuration file {}: {}".format(conf,e))
return True
if xconf.__resolve_iterators__():
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_conf__)")
self.logger.error(" Cannot resolve iterators in configuration file {}: {}".format(conf,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot resolve iterators in configuration file {}: {}".format(conf,e))
return True
if parameters is None:
for n in xconf.methods_txt:
self.methods_txt[n] = xconf.methods_txt[n]
return False
else:
if type(parameters) is str:
parameters = [ parameters ]
elif type(parameters) is list or type(parameters) is tuple:
if reduce(lambda x,y: x and type(y) is str, parameters, True): pass
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_conf__)")
self.logger.error(" Not all parameters {} are strings".format(parameters))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Not all parameters {} are strings".format(parameters))
return True
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_conf__)")
self.logger.error(" Incorrect type of imported parameters. It should be string or list/tuple of string. {} is given".format(type(parameters)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Incorrect type of exported parameters. It should be string or list/tuple of string. {} is given".format(type(parameters)))
return True
for param in parameters:
if not param in xconf.methods_txt:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_conf__)")
self.logger.error(" Cannot import parameter {} from the configuration file {}: there is no such parameter".format(param,conf) )
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Cannot import parameter {} from the configuration file {}: there is no such parameter".format(param,conf) )
#return True # << Not sure what is the best option here
continue
self.methods_txt[param] = xconf.methods_txt[param]
return False
def __read_db__(self,dburl,record,parameters=None):
try:
d = db(dburl,mode="ro")
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_db__)")
self.logger.error(" Cannot open data base {}: {}".format(dburl,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot open data base {}: {}".format(dburl,e))
return True
if parameters is None:
try:
xtree = d[record]
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_db__)")
self.logger.error(" Cannot pool record {} from data base {}: {}".format(record, dburl,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot pool record {} from data base {}: {}".format(record, dburl,e))
return True
for n in xtree:
if type(xtree[n]) is str or type(xtree[n]) is str:
self.methods_txt[n] = xtree[n]
else:
self.methods[n] = xtree[n]
return False
else:
if type(parameters) is str:
parameters = [ parameters ]
elif type(parameters) is list or type(parameters) is tuple:
if reduce(lambda x,y: x and type(y) is str, parameters, True): pass
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_db__)")
self.logger.error(" Not all parameters {} are strings".format(parameters))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Not all parameters {} are strings".format(parameters))
return True
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_db__)")
self.logger.error(" Incorrect type of imported parameters. It should be string or list/tuple of string. {} is given".format(type(parameters)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Incorrect type of exported parameters. It should be string or list/tuple of string. {} is given".format(type(parameters)))
return True
for param in parameters:
try:
xtree = d[record,param]
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __read_db__)")
self.logger.error(" Cannot pool param {} from record {} in data base {}: {}".format(param, record, dburl,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot pool param {} from record {} in data base {}: {}".format(param, record, dburl,e))
return True
for n in xtree:
if type(xtree[n]) is str or type(xtree[n]) is str:
self.methods_txt[n] = xtree[n]
else:
self.methods[n] = xtree[n]
return False
def gethash(self, name=''):
if name == "/":
ret=""
for n in self.methods:
ret += self.gethash(name = n) if ret == "" else ":"+self.gethash(name = n)
return hashlib.sha256( ret.encode() ).hexdigest()
if not name in self.methods: return None
if isinstance(self.methods[name], tree):
achash=""
for n in self.methods[name].dict():
achash += ":"+self.gethash(name+self.hsymbol+n)
return achash[1:]
elif name in self.hashspace:
return self.hashspace[name]
elif self.is_lambda(self.methods[name]):
if name in self.methods_txt:
self.hashspace[name] = hashlib.sha1(self.methods_txt[name][0].encode()).hexdigest()
return self.hashspace[name]
else:
self.logger.warning(" > LAMBDA function not in namespace")
self.hashspace[name] = hashlib.sha1(str(self.methods[name]).encode()).hexdigest()
return self.hashspace[name]
else:
self.hashspace[name] = hashlib.sha1(str(self.methods[name]).encode()).hexdigest()
return self.hashspace[name]
return None
# Functions for tree like behavior.
# They mostly redirect calls to self.methods tree
def __setitem__(self, key, value): self.methods.__setitem__(key, value)
def __getitem__(self, key) :
if key[0] == "#":
return self.gethash(key[1:])
if not self.methods.__contains__(key) and self.methods_txt.__contains__(key):
if self.generate(var=key):
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __getitem__)")
self.logger.error(" Cannot resolve generate key {}".format(key))
self.logger.error("----------------------------------------------------")
if self.pure : raise KeyError("Cannot resolve generate key {}".format(key))
return None
return self.methods.__getitem__(key)
def __contains__(self,key) :
return self.methods.__contains__(key) or self.methods_txt.__contains__(key)
def __delitem__(self, key) : self.methods.__delitem__(key)
def __iter__(self):
for name in self.methods.__iter__(): yield name
def check(self, key) : return self.methods.check(key)
def dict(self):
for name in self.methods.dict():yield name
def __namefilter__(self,name,flt):
return reduce(lambda x,y: x or name.startswith(y), flt, False)
# Functions that parse and convert text values into objects
def __resolve_iterators__(self):
"""
resolves all iterators
"""
for itr in self.iterate:
append_methods_txt = tree(hsymbol=self.hsymbol)
value = self.resolve_name(self.localcontext[self.target].methods, self.target, self.iterate[itr], prohibit=[])
if value is None:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __resolve_iterators__)")
self.logger.error(" Cannot resolve iterator {}".format(itr))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot resolve iterator {}".format(itr))
return True
try:
exec("{}.itrvalue={}".format(self.target,value), self.localcontext)
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in __resolve_iterators__)")
self.logger.error(" Cannot execute operation {}.itrvalue={}: {}".format(self.target,value,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot execute operation {}.itrvalue={}: {}".format(self.target,value,e))
return True
for name in self.methods_txt:
if itr in name or itr in self.methods_txt[name][0]:
for iv in self.itrvalue:
append_methods_txt[name.replace(itr,"{}".format(iv))] = \
self.methods_txt[name][0].replace(itr,"{}".format(iv)), self.methods_txt[name][1].replace(itr,"{}".format(iv))
else:
append_methods_txt[name] = self.methods_txt[name]
self.methods_txt = append_methods_txt
#del self.itrvalue
# #cleanup text tree
# for name in self.methods_txt:
# if self.isymbol in name or self.isymbol in self.methods_txt[name]:
# del self.methods_txt[name]
new_methods_txt = tree(hsymbol=self.hsymbol)
for name in self.methods_txt:
if self.isymbol not in name and self.isymbol not in self.methods_txt[name]:
new_methods_txt[name] = self.methods_txt[name]
self.methods_txt = new_methods_txt
return False
def builder(self, tree, target, item, delimiter, prohibit):
"""
Finds and resolves build expressions for links and strings
"""
if delimiter not in item: return item
result = ''
copir = item.split(delimiter)
if len(copir)%2: copir += [None]
for prefix,var in zip(copir[::2],copir[1::2]):
if prefix is not None: result += prefix
if var is None: continue
if not var in tree and var in self.methods_txt:
if self.__namefilter__(var, prohibit):
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Parameter \'{}\' wasn't resolved and it is prohibited.".format(var))
self.logger.error("----------------------------------------------------")
if self.pure: raise TypeError("Parameter \'{}\' wasn't resolved and it is prohibited.".format(var))
return None
self.stackcnt += 1
if self.stackcnt > 100:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Exceed number of stack operation (101).")
self.logger.error("----------------------------------------------------")
if self.pure: raise ValueError("Exceed number of stack operation (101).")
return None
self.generate(var,target=self.target,localcontext=self.localcontext, prohibit=prohibit)
elif not var in tree and not var in self.methods_txt:
short_var = var.split(self.hsymbol)
for sid in range(len(short_var)-1,1,-1):
nvar = self.hsymbol.join(short_var[:sid])
if nvar in self.methods_txt:
if self.__namefilter__(nvar, prohibit):
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Parameter \'{}\' wasn't resolved because \'{}\' is prohibited.".format(var,nvar))
self.logger.error("----------------------------------------------------")
if self.pure: raise TypeError("Parameter \'{}\' wasn't resolved because \'{}\' is prohibited.".format(var,nvar))
return None
self.stackcnt += 1
if self.stackcnt > 100:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Exceed number of stack operation (101).")
self.logger.error("----------------------------------------------------")
if self.pure: raise ValueError("Exceed number of stack operation (101).")
return None
self.generate(nvar,target=self.target,localcontext=self.localcontext, prohibit=prohibit)
break
lmdcheck = self.is_lambda(tree[var])
if lmdcheck or delimiter is self.refsymbol:
result += target+".methods[\""+var+"\"]"
elif delimiter is self.strefsymbol:
if var in self.methods_txt:
result += self.methods_txt[var][0]
else:
result += str(tree[var])
elif delimiter is self.refhash:
result += "\'\\\'{}\\\'\'".format(self.gethash(var))
else:
return None
self.dependences.append(var)
return result
def resolve_name(self, tree, target, item, prohibit):
"""
Resolves links, string and hashes in RHS of parameters
"""
# Resolve links First
# then Resolve strings
# and then Resolve hashs
result = item
for delimiter in self.refsymbol, self.strefsymbol, self.refhash:
bld = self.builder(tree, target, result, delimiter , prohibit=prohibit)
if bld is None: return None
result = bld
return str(result)
def generate(self, var=None, target=None, localcontext = None, prohibit=None, text=False):
"""
generates python objects in self.methods tree based on records in self.methods_txt tree
@opt var - variable or list of variables which should be generated
@opt target - string of variable name which will be generated (need for lambda(s)),
@opt localcontext- local name space, usually = globals() or locals()
@opt prohibit - list of name which shouldn't be generated
@opt text - bool If true populate tree by strings not actual values
"""
self.target = self.dtarget if target is None else target
if not localcontext is None: self.localcontext = localcontext
elif not self.dlocalctx is None: self.localcontext = self.dlocalctx
else: self.localcontext = globals()
if not self.target in self.localcontext:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Target object \'{}\' is not found in context".format(self.target))
self.logger.error("----------------------------------------------------")
if self.pure: raise ValueError("Target object \'{}\' is not found in context".format(self.target))
return True
if not prohibit is None:
if type(prohibit) is str: prohibit = [ prohibit ]
elif type(prohibit) is list or type(prohibit) is tuple:
if reduce(lambda x,y: x and type(y) is str, prohibit, True) : pass
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" One of entrances of the Prohibit option has wrong type.")
self.logger.error("----------------------------------------------------")
if self.pure: raise TypeError("One of entrances of the Prohibit option has wrong type.")
return True
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Prohibit option has wrong type \'{}\'.".format(type(prohibit)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Prohibit option has wrong type \'{}\'.".format(type(prohibit)))
return True
else : prohibit = []
if not self.iterate_res :
self.iterate_res = True
if self.__resolve_iterators__():
self.iterate_res = False
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Cannot resolve iterators")
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot resolve iterators")
return True
if var is None: var = self.methods_txt
elif type(var) is list or type(var) is tuple: pass
elif type(var) is str : var = [ var ]
else:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Variable option has wrong type \'{}\'; it should be a string or a list of strings or None.".format(type(var)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Variable option has incorrect type \'{}\'.".format(type(prohibit)))
return True
for name in var:
if not type(name) is str:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Variable name is not string: type \'{}\' is given.".format(type(name)))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Variable name is not string: type \'{}\' is given.".format(type(name)))
return True
if self.__namefilter__(name, prohibit):
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Resolving a name \'{}\' is prohibited.".format(name))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Resolving a name \'{}\' is prohibited.".format(name))
return True
if not name in self.methods_txt:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Cannot find parameter name \'{}\' is the text tree.".format(name))
self.logger.error("----------------------------------------------------")
if self.pure : raise TypeError("Cannot find parameter name \'{}\' is the text tree.".format(name))
return True
value = self.methods_txt[name]
if isinstance(value, tree):
for n in value:
self.generate(name+n,target=self.target, localcontext = self.localcontext, prohibit=prohibit, text=text)
else:
value = value[0]
self.dependences = []
if text:
if not self.hashspace.check(name):
self.hashspace[name] = hashlib.sha1(value.encode()).hexdigest()
pass
try:
exec("{}.methods[\'{}\']=\"{}\"".format(self.target,name,re.sub(r"\\", "\\\\", re.sub(r"\"","\\\"", re.sub("\'","\\\'", value) ) )), self.localcontext)
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Cannot execute operation {}[\'{}\']=\"{}\": {}".format(
target, name,
re.sub(r"\\", "\\\\", re.sub(r"\"","\\\"", re.sub("\'","\\\'", value))),e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot execute operation {}[\'{}\']=\"{}\": {}".format(target,name,re.sub(r"\\", "\\\\", re.sub(r"\"","\\\"", re.sub("\'","\\\'", value))),e))
return True
else:
value = self.resolve_name(self.localcontext[self.target].methods, self.target,value,prohibit=prohibit)
try:
exec("{}.methods[\'{}\']={}".format(self.target,name,value), self.localcontext)
except BaseException as e:
self.logger.error("----------------------------------------------------")
self.logger.error(" METHODS ERROR in generate)")
self.logger.error(" Cannot execute operation {}[\'{}\']={}: {}".format(self.target,name,value,e))
self.logger.error("----------------------------------------------------")
if self.pure : raise ValueError("Cannot execute operation {}[\'{}\']={}: {}".format(self.target,name,value,e))
return True
#if not self.hashspace.check(name):
#self.hashspace[name] = self.gethash(name)
#self.hashspace[name] = self.gethash(name)
## Update hash everytime then parameter change
## NOTE: it doesn't help with data set up outside methods
self.hashspace[name] = self.gethash(name)#hashlib.sha1(str(self.methods[name])).hexdigest()
for dep in self.dependences:
self.hashspace[name] += ":"+self.gethash(dep)
self.dependences = []
self.logger.debug( " > % 76s : OK"%(name))
return False
def printhelp(self,names=None):
#if names is None:
#names = self.methods_txt
ret = "\nParameters:\n"
for p,k,s in self.methods_txt.printnames():
#if not p in names: continue
if k is None:
ret += "{}\n".format(p)
else:
refadd = "{} = {}".format(p,self.methods_txt[k][0])
if len(refadd) < 33: refadd += " "*(33-len(refadd)) + " ; {} \n".format(self.methods_txt[k][1])
else :
c = p[len(s)]
if c == "|": c = s+c
else : c = s
refadd += "\n"+c+" "*(33-len(c)) + " ; {} \n".format(self.methods_txt[k][1])
ret += refadd
ret += "\n"
return ret
def genhelp(self):
"""
This is a generator!
Generates triples: parameter,value,help message
"""
for n in self.methods_txt:
yield n,self.methods_txt[n][0],self.methods_txt[n][1]
if __name__ == "__main__":
#CHECK LOGGER
logging.basicConfig(format='%(asctime)s: %(levelname)-8s:%(message)s', level=logging.INFO)
if len(sys.argv) < 2:
print("USEAGE: python simtoolkit/methods.py model-fileformats/example-generalsyntax.stkconf")
print("\n\n=============================\n")
print( "=== JUST A SIMPLE EXAMPLE ===")
m = methods("""
/{
A very simple example of configuration
}
/parameter1 = 12 ; first parameter in the model
/group/parameter1 = 11 ; first parameter in the group
/group/parameter2 = .5 ; second parameter in the group
/group2 {
/parameter1 = 144 ; parameter 1 in group 2
/parameter2 = 143 ; parameter 2 in group 2
}
/group3 {
/subgroup {
/p1 = 1 ; parameter 1 of subgroup in group 3
/p2 = 2 ; parameter 2 of subgroup in group 3
}
/p1 = 3 ; parameter 1 in group 3
/p2 = 4 ; parameter 2 in group 3
}
/types/{
/int = 12 ; this is an integer
/float = 2.5 ; this is an float
/bool = True ; this is a boolean parameter
/str = "I love python" ; string will be a string
/list = [0,1,2,3,4] ; this is a standard list
/tuple = (0,1,2,3,4) ; a tuple
/dict = {'a':1,'b':2,'c':3} ; but a dictionary will be converted in a subtree
/func = lambda x: x**2 ; this is a function.
/func = lambda x: x**2 ; this is a function.
}
/2by2is = 2*2 ; calculates 2*2
/xlst = [ x for x in range(10) if x%3 != 0 ] ; /xlst is [1, 2, 4, 5, 7, 8]
/L = range( @/parameter1@ ) ; equals to range(12)
/P = @/xlst@[::@/2by2is@] ; P uses a list and filters out values , i.e [1, 7]
/calc = [ @/types/func@(x) for x in \ ; the range depends upon /group/parameter1
range(@/group/parameter1@) if x%3 != 0 ] ; i.e /calc is [1, 4, 16, 25, 49, 64, 100]
/calc2 = [ @/types/func@(x) for x in @/xlst@ ] ; /calc is [1, 4, 16, 25, 49, 64]
/MyGroup/Item` x = range(3) ` = `x` ; Iteratable item
/STIMULI/Stimulus` x = [1,2,3]`-and-`y=[5,6,7]`/{
/ParameterX = `x` ; Parameter X of the Stimulus `x`-and-`y`
/ParameterY = `y` ; Parameter Y of the Stimulus `y`-and-`y`
/CombinationXandY = `x`*`y` ; Combined param of the Stimulus `x`-and-`y`
}
/hash = #/types/str# ; hash of string
"""
,"m",globals())
elif len(sys.argv) < 3:
with open(sys.argv[1]) as fd:
m = methods(fd,"m",globals())
else:
with open(sys.argv[1]) as fd:
m = methods(fd,"m",globals(),argvs=sys.argv[2:])
print()
print(m.mainmessage.replace("\n", "\nMESSAGE: "))
print()
print("THE TEXT TREE (BEFORE FIRST RESOLVING)")
for p,k,s in m.methods_txt.printnames():
print(p, "" if k is None else m.methods_txt[k] )
print()
print("THE ITEERATIORS")
for p in m.iterate:
print(p,m.iterate[p])
#help(m)
print("#====================================#")
print("# ERROR CHECKS #")
m.generate(target="x",localcontext=globals())
m.generate(prohibit=10)
m.generate(prohibit=['/a','/b',30])
m.generate(10)
m.generate([12,'/parameter1','/goup2/parameter2'])
m.generate('/a',prohibit=['/a','/b'])
m.generate('/a/b/c',prohibit=['/a','/b'])
m.generate('/ppy')
print("#====================================#")
print()
print("THE TEXT TREE (AFTER SOME RESOLVING)")
for p,k,s in m.methods_txt.printnames():
print(p, "" if k is None else m.methods_txt[k] )
print()
print("#====================================#")
print("# RESOLVING PARAMETERS #")
m.generate("/parameter1")
m.generate(["/group","/group2","/group3"])
m.generate("/calc2")
m.generate("/types")
m.generate("/MyGroup")
m.generate("/MultiIter")
m.generate(["/L","/P"])
m.generate(["/2by2is",'/calc',"/Multiline/Item"])
m.generate("/str")
m.generate("/hash")
print("#====================================#")
print()
print("#====================================#")
print("# THE VALUE TREE #")
print("# (/STIMULI weren't resolved yet) #")
for p,k,s in m.methods.printnames():
print(p, "" if k is None else m.methods[k] )
print("#====================================#")
print()
print("#====================================#")
print("# THE COMPLETE TREE #")
m.generate()
for p,k,s in m.methods.printnames():
print(p, "" if k is None else m.methods[k] )
print("#====================================#")
print("#====================================#")
print("# RESOLVING AS TEXT #")
m.generate(text=True)
print("THE TEXT VALUE TREE")
for p,k,s in m.methods.printnames():
print(p, "" if k is None else "{}[{}]".format(m.methods[k],type(m.methods[k])) )
print("#====================================#")
print()
print("#====================================#")
print("# HELP! #")
print(m.printhelp())
print("#====================================#")
print()
##>> Remove a /STIMULI to show resolving in line
del m["/STIMULI"]
print("=================================================================================")
print("# RESOLVED INLINE #")
print("# THE STIMULI TREE : #")
for p in m["/STIMULI"]:
print("% 55s :"%("/STIMULI"+p),m["/STIMULI"+p])
print("=================================================================================")
print()