Update Js2Py 0.70 (92250a4) → 0.74 (2e017b8).

CHANGES.md

@@ -9,6 +9,7 @@
 * Update diskcache 5.1.0 (40ce0de) to 5.4.0 (1cb1425)
 * Update feedparser 6.0.1 (98d189fa) to 6.0.10 (5fcb3ae)
 * Update humanize 3.5.0 (b6b0ea5) to 4.0.0 (a1514eb)
+* Update Js2Py 0.70 (92250a4) to 0.74 (2e017b8)
 * Update profilehooks module 1.12.0 (3ee1f60) to 1.12.1 (c3fc078)
 * Update Rarfile 4.0 (55fe778) to 4.1a1 (8a72967)
 * Update UnRar x64 for Windows 6.11 to 6.20

lib/js2py/constructors/jsdate.py

@@ -118,20 +118,39 @@ class PyJsDate(PyJs):
 
 
 def parse_date(py_string):  # todo support all date string formats
+    date_formats = (
+        "%Y-%m-%d",
+        "%m/%d/%Y",
+        "%b %d %Y",
+    )
+    # Supports these hour formats and with or hour.
+    hour_formats = (
+        "T%H:%M:%S.%f",
+        "T%H:%M:%S",
+    ) + ('',)
+    # Supports with or without Z indicator.
+    z_formats = ("Z",) + ('',)
+    supported_formats = [
+        d + t + z
+        for d in date_formats
+        for t in hour_formats
+        for z in z_formats
+    ]
+    for date_format in supported_formats:
         try:
-        try:
+            dt = datetime.datetime.strptime(py_string, date_format)
-            dt = datetime.datetime.strptime(py_string, "%Y-%m-%dT%H:%M:%S.%fZ")
+        except ValueError:
-        except:
+            continue
-            dt = datetime.datetime.strptime(py_string, "%Y-%m-%dT%H:%M:%SZ")
+        else:
             return MakeDate(
                 MakeDay(Js(dt.year), Js(dt.month - 1), Js(dt.day)),
                 MakeTime(
                     Js(dt.hour), Js(dt.minute), Js(dt.second),
                     Js(dt.microsecond // 1000)))
-    except:
+
     raise MakeError(
         'TypeError',
-            'Could not parse date %s - unsupported date format. Currently only supported format is RFC3339 utc. Sorry!'
+        'Could not parse date %s - unsupported date format. Currently only supported formats are RFC3339 utc, ISO Date, Short Date, and Long Date. Sorry!'
         % py_string)
 
 

lib/js2py/constructors/time_helpers.py

@@ -36,8 +36,8 @@ def DaylightSavingTA(t):
         return t
     try:
         return int(
-            LOCAL_ZONE.dst(datetime.datetime.utcfromtimestamp(
+            LOCAL_ZONE.dst(datetime.datetime(1970, 1, 1) + datetime.timedelta(
-                t // 1000)).seconds) * 1000
+                seconds=t // 1000)).seconds) * 1000
     except:
         warnings.warn(
             'Invalid datetime date, assumed DST time, may be inaccurate...',

lib/js2py/evaljs.py

@@ -53,7 +53,7 @@ def write_file_contents(path_or_file, contents):
     if hasattr(path_or_file, 'write'):
         path_or_file.write(contents)
     else:
-        with open(path_as_local(path_or_file), 'w') as f:
+        with codecs.open(path_as_local(path_or_file), "w", "utf-8") as f:
             f.write(contents)
 
 
@@ -238,6 +238,10 @@ class EvalJs(object):
         self.execute_debug(code)
         return self['PyJsEvalResult']
 
+    @property
+    def context(self):
+        return self._context
+    
     def __getattr__(self, var):
         return getattr(self._var, var)
 
@@ -268,14 +272,3 @@ class EvalJs(object):
                 else:
                     sys.stderr.write('EXCEPTION: ' + str(e) + '\n')
                 time.sleep(0.01)
-
-
-#print x
-
-if __name__ == '__main__':
-    #with open('C:\Users\Piotrek\Desktop\esprima.js', 'rb') as f:
-    #    x = f.read()
-    e = EvalJs()
-    e.execute('square(x)')
-    #e.execute(x)
-    e.console()

lib/js2py/host/console.py

@@ -6,7 +6,7 @@ def console():
 
 @Js
 def log():
-    print(arguments[0])
+    print(" ".join(repr(element) for element in arguments.to_list()))
 
 console.put('log', log)
 console.put('debug', log)

lib/js2py/internals/__init__.py

@@ -0,0 +1 @@
+from .seval import eval_js_vm

lib/js2py/internals/base.py

@@ -602,11 +602,12 @@ class PyJsDate(PyJs):
 
     # todo fix this problematic datetime part
     def to_local_dt(self):
-        return datetime.datetime.utcfromtimestamp(
+        return datetime.datetime(1970, 1, 1) + datetime.timedelta(
-            self.UTCToLocal(self.value) // 1000)
+            seconds=self.UTCToLocal(self.value) // 1000)
 
     def to_utc_dt(self):
-        return datetime.datetime.utcfromtimestamp(self.value // 1000)
+        return datetime.datetime(1970, 1, 1) + datetime.timedelta(
+            seconds=self.value // 1000)
 
     def local_strftime(self, pattern):
         if self.value is NaN:

lib/js2py/internals/constructors/time_helpers.py

@@ -38,8 +38,8 @@ def DaylightSavingTA(t):
         return t
     try:
         return int(
-            LOCAL_ZONE.dst(datetime.datetime.utcfromtimestamp(
+            LOCAL_ZONE.dst(datetime.datetime(1970, 1, 1) + datetime.timedelta(
-                t // 1000)).seconds) * 1000
+                seconds=t // 1000)).seconds) * 1000
     except:
         warnings.warn(
             'Invalid datetime date, assumed DST time, may be inaccurate...',

lib/js2py/internals/opcodes.py

@@ -798,7 +798,7 @@ OP_CODES = {}
 g = ''
 for g in globals():
     try:
-        if not issubclass(globals()[g], OP_CODE) or g is 'OP_CODE':
+        if not issubclass(globals()[g], OP_CODE) or g == 'OP_CODE':
             continue
     except:
         continue

lib/js2py/internals/prototypes/jsstring.py

@@ -22,6 +22,11 @@ def replacement_template(rep, source, span, npar):
                 res += '$'
                 n += 2
                 continue
+            elif rep[n + 1] == '&':
+                # replace with matched string
+                res += source[span[0]:span[1]]
+                n += 2
+                continue
             elif rep[n + 1] == '`':
                 # replace with string that is BEFORE match
                 res += source[:span[0]]

lib/js2py/node_import.py

@@ -4,10 +4,13 @@ import subprocess, os, codecs, glob
 from .evaljs import translate_js, DEFAULT_HEADER
 from .translators.friendly_nodes import is_valid_py_name
 import six
+import tempfile
+import hashlib
+import random
 
 DID_INIT = False
-DIRNAME = os.path.dirname(os.path.abspath(__file__))
+DIRNAME = tempfile.mkdtemp()
-PY_NODE_MODULES_PATH = os.path.join(DIRNAME, 'py_node_modules')
+PY_NODE_MODULES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'py_node_modules')
 
 
 def _init():
@@ -72,8 +75,10 @@ def _get_and_translate_npm_module(module_name, include_polyfill=False, update=Fa
     if not os.path.exists(os.path.join(PY_NODE_MODULES_PATH,
                                        module_filename)) or update:
         _init()
-        in_file_name = 'tmp0in439341018923js2py.js'
+        module_hash = hashlib.sha1(module_name.encode("utf-8")).hexdigest()[:15]
-        out_file_name = 'tmp0out439341018923js2py.js'
+        version = random.randrange(10000000000000)
+        in_file_name = 'in_%s_%d.js' % (module_hash, version)
+        out_file_name = 'out_%s_%d.js' % (module_hash, version)
         code = ADD_TO_GLOBALS_FUNC
         if include_polyfill:
             code += "\n;require('babel-polyfill');\n"
@@ -106,7 +111,7 @@ def _get_and_translate_npm_module(module_name, include_polyfill=False, update=Fa
         with codecs.open(os.path.join(DIRNAME, out_file_name), "r",
                          "utf-8") as f:
             js_code = f.read()
-        os.remove(os.path.join(DIRNAME, out_file_name))
+        print("Bundled JS library dumped at: %s" % os.path.join(DIRNAME, out_file_name))
         if len(js_code) < 50:
             raise RuntimeError("Candidate JS bundle too short - likely browserify issue.")
         js_code += GET_FROM_GLOBALS_FUNC

lib/js2py/prototypes/jsstring.py

@@ -17,6 +17,11 @@ def replacement_template(rep, source, span, npar):
                 res += '$'
                 n += 2
                 continue
+            elif rep[n + 1] == '&':
+                # replace with matched string
+                res += source[span[0]:span[1]]
+                n += 2
+                continue
             elif rep[n + 1] == '`':
                 # replace with string that is BEFORE match
                 res += source[:span[0]]

lib/js2py/translators/translating_nodes.py

@@ -14,26 +14,36 @@ if six.PY3:
 LINE_LEN_LIMIT = 400  #  200  # or any other value - the larger the smaller probability of errors :)
 
 
-class ForController:
+class LoopController:
     def __init__(self):
-        self.inside = [False]
+        self.update = [""]
-        self.update = ''
+        self.label_to_update_idx = {}
 
-    def enter_for(self, update):
+    def enter(self, update=""):
-        self.inside.append(True)
+        self.update.append(update)
-        self.update = update
 
-    def leave_for(self):
+    def leave(self):
-        self.inside.pop()
+        self.update.pop()
+
+    def get_update(self, label=None):
+        if label is None:
+            return self.update[-1]
+        if label not in self.label_to_update_idx:
+            raise SyntaxError("Undefined label %s" % label)
+        if self.label_to_update_idx[label] >= len(self.update):
+            raise SyntaxError("%s is not a iteration statement label?" % label)
+        return self.update[self.label_to_update_idx[label]]
+
+    def register_label(self, label):
+        if label in self.label_to_update_idx:
+            raise SyntaxError("label %s already used")
+        self.label_to_update_idx[label] = len(self.update)
+
+    def deregister_label(self, label):
+        del self.label_to_update_idx[label]
 
-    def enter_other(self):
-        self.inside.append(False)
 
-    def leave_other(self):
-        self.inside.pop()
 
-    def is_inside(self):
-        return self.inside[-1]
 
 
 class InlineStack:
@@ -86,9 +96,10 @@ class ContextStack:
 
 
 def clean_stacks():
-    global Context, inline_stack
+    global Context, inline_stack, loop_controller
     Context = ContextStack()
     inline_stack = InlineStack()
+    loop_controller = LoopController()
 
 
 def to_key(literal_or_identifier):
@@ -374,9 +385,14 @@ def BreakStatement(type, label):
 
 def ContinueStatement(type, label):
     if label:
-        return 'raise %s("Continued")\n' % (get_continue_label(label['name']))
+        maybe_update_expr = loop_controller.get_update(label=label['name'])
+        continue_stmt = 'raise %s("Continued")\n' % (get_continue_label(label['name']))
     else:
-        return 'continue\n'
+        maybe_update_expr = loop_controller.get_update()
+        continue_stmt = "continue\n"
+    if maybe_update_expr:
+        return "# continue update\n%s\n%s" % (maybe_update_expr, continue_stmt)
+    return continue_stmt
 
 
 def ReturnStatement(type, argument):
@@ -393,24 +409,28 @@ def DebuggerStatement(type):
 
 
 def DoWhileStatement(type, body, test):
-    inside = trans(body) + 'if not %s:\n' % trans(test) + indent('break\n')
+    loop_controller.enter()
+    body_code = trans(body)
+    loop_controller.leave()
+    inside = body_code + 'if not %s:\n' % trans(test) + indent('break\n')
     result = 'while 1:\n' + indent(inside)
     return result
 
 
 def ForStatement(type, init, test, update, body):
-    update = indent(trans(update)) if update else ''
+    update = trans(update) if update else ''
     init = trans(init) if init else ''
     if not init.endswith('\n'):
         init += '\n'
     test = trans(test) if test else '1'
+    loop_controller.enter(update)
     if not update:
         result = '#for JS loop\n%swhile %s:\n%s%s\n' % (
             init, test, indent(trans(body)), update)
     else:
         result = '#for JS loop\n%swhile %s:\n' % (init, test)
-        body = 'try:\n%sfinally:\n    %s\n' % (indent(trans(body)), update)
+        result += indent("%s# update\n%s\n" % (trans(body), update))
-        result += indent(body)
+    loop_controller.leave()
     return result
 
 
@@ -429,7 +449,9 @@ def ForInStatement(type, left, right, body, each):
         name = left['name']
     else:
         raise RuntimeError('Unusual ForIn loop')
+    loop_controller.enter()
     res += indent('var.put(%s, PyJsTemp)\n' % repr(name) + trans(body))
+    loop_controller.leave()
     return res
 
 
@@ -445,20 +467,23 @@ def IfStatement(type, test, consequent, alternate):
 
 def LabeledStatement(type, label, body):
     # todo consider using smarter approach!
+    label_name = label['name']
+    loop_controller.register_label(label_name)
     inside = trans(body)
+    loop_controller.deregister_label(label_name)
     defs = ''
     if is_iteration_statement(body) and (inside.startswith('while ') or inside.startswith(
             'for ') or inside.startswith('#for')):
         # we have to add contine label as well...
         # 3 or 1 since #for loop type has more lines before real for.
         sep = 1 if not inside.startswith('#for') else 3
-        cont_label = get_continue_label(label['name'])
+        cont_label = get_continue_label(label_name)
         temp = inside.split('\n')
         injected = 'try:\n' + '\n'.join(temp[sep:])
         injected += 'except %s:\n    pass\n' % cont_label
         inside = '\n'.join(temp[:sep]) + '\n' + indent(injected)
         defs += 'class %s(Exception): pass\n' % cont_label
-    break_label = get_break_label(label['name'])
+    break_label = get_break_label(label_name)
     inside = 'try:\n%sexcept %s:\n    pass\n' % (indent(inside), break_label)
     defs += 'class %s(Exception): pass\n' % break_label
     return defs + inside
@@ -553,7 +578,11 @@ def VariableDeclaration(type, declarations, kind):
 
 
 def WhileStatement(type, test, body):
-    result = 'while %s:\n' % trans(test) + indent(trans(body))
+    test_code = trans(test)
+    loop_controller.enter()
+    body_code = trans(body)
+    loop_controller.leave()
+    result = 'while %s:\n' % test_code + indent(body_code)
     return result
 
 

lib/js2py/translators/translator.py

@@ -55,16 +55,19 @@ def dbg(x):
     """does nothing, legacy dummy function"""
     return ''
 
+# Another way of doing that would be with my auto esprima translation but its much slower:
+# parsed = esprima.parse(js).to_dict()
+def pyjsparser_parse_fn(code):
+    parser = pyjsparser.PyJsParser()
+    return parser.parse(code)
 
-def translate_js(js, HEADER=DEFAULT_HEADER, use_compilation_plan=False):
+def translate_js(js, HEADER=DEFAULT_HEADER, use_compilation_plan=False, parse_fn=pyjsparser_parse_fn):
     """js has to be a javascript source code.
        returns equivalent python code."""
     if use_compilation_plan and not '//' in js and not '/*' in js:
         return translate_js_with_compilation_plan(js, HEADER=HEADER)
-    parser = pyjsparser.PyJsParser()
+
-    parsed = parser.parse(js)  # js to esprima syntax tree
+    parsed = parse_fn(js)
-    # Another way of doing that would be with my auto esprima translation but its much slower and causes import problems:
-    # parsed = esprima.parse(js).to_dict()
     translating_nodes.clean_stacks()
     return HEADER + translating_nodes.trans(
         parsed)  # syntax tree to python code

lib/js2py/utils/injector.py

@@ -26,17 +26,19 @@ def fix_js_args(func):
         return func
     code = append_arguments(six.get_function_code(func), ('this', 'arguments'))
 
-    return types.FunctionType(
+    result = types.FunctionType(
         code,
         six.get_function_globals(func),
         func.__name__,
         closure=six.get_function_closure(func))
+    return result
 
 
 def append_arguments(code_obj, new_locals):
     co_varnames = code_obj.co_varnames  # Old locals
     co_names = code_obj.co_names  # Old globals
-    co_names += tuple(e for e in new_locals if e not in co_names)
+    new_args = tuple(e for e in new_locals if e not in co_names)
+    co_names += new_args
     co_argcount = code_obj.co_argcount  # Argument count
     co_code = code_obj.co_code  # The actual bytecode as a string
 
@@ -76,26 +78,51 @@ def append_arguments(code_obj, new_locals):
     names_to_varnames = dict(
         (co_names.index(name), varnames.index(name)) for name in new_locals)
 
+    is_new_bytecode = sys.version_info >= (3, 11)
     # Now we modify the actual bytecode
     modified = []
+    drop_future_cache = False
     for inst in instructions(code_obj):
+        if is_new_bytecode and inst.opname == "CACHE":
+            assert inst.arg == 0
+            if not drop_future_cache:
+                modified.extend(write_instruction(inst.opcode, inst.arg))
+            else:
+                # We need to inject NOOP to not break jumps :(
+                modified.extend(write_instruction(dis.opmap["NOP"], 0))
+
+            continue
         op, arg = inst.opcode, inst.arg
         # If the instruction is a LOAD_GLOBAL, we have to check to see if
         # it's one of the globals that we are replacing. Either way,
         # update its arg using the appropriate dict.
+        drop_future_cache = False
         if inst.opcode == LOAD_GLOBAL:
-            if inst.arg in names_to_varnames:
+            idx = inst.arg
+            if is_new_bytecode:
+                idx = idx // 2
+            if idx in names_to_varnames:
                 op = LOAD_FAST
-                arg = names_to_varnames[inst.arg]
+                arg = names_to_varnames[idx]
-            elif inst.arg in name_translations:
+                # Cache is not present after LOAD_FAST and needs to be removed.
-                arg = name_translations[inst.arg]
+                drop_future_cache = True
+            elif idx in name_translations:
+                tgt = name_translations[idx]
+                if is_new_bytecode:
+                    tgt = 2*tgt + (inst.arg % 2)
+                arg = tgt
             else:
-                raise ValueError("a name was lost in translation")
+                raise(ValueError("a name was lost in translation last instruction %s" % str(inst)))
         # If it accesses co_varnames or co_names then update its argument.
         elif inst.opcode in opcode.haslocal:
             arg = varname_translations[inst.arg]
         elif inst.opcode in opcode.hasname:
+            # for example STORE_GLOBAL
             arg = name_translations[inst.arg]
+        elif is_new_bytecode and inst.opcode in opcode.hasfree:
+            # Python 3.11+ adds refs at the end (after locals), for whatever reason...
+            if inst.argval not in code_obj.co_varnames[:code_obj.co_argcount]:  # we do not need to remap existing arguments, they are not shifted by new ones.
+                arg = inst.arg + len(new_locals)
         modified.extend(write_instruction(op, arg))
     if six.PY2:
         code = ''.join(modified)
@@ -113,23 +140,26 @@ def append_arguments(code_obj, new_locals):
                 code_obj.co_filename, code_obj.co_name,
                 code_obj.co_firstlineno, code_obj.co_lnotab,
                 code_obj.co_freevars, code_obj.co_cellvars)
-
     # Done modifying codestring - make the code object
     if hasattr(code_obj, "replace"):
         # Python 3.8+
-        return code_obj.replace(
+        code_obj = code_obj.replace(
             co_argcount=co_argcount + new_locals_len,
             co_nlocals=code_obj.co_nlocals + new_locals_len,
             co_code=code,
             co_names=names,
             co_varnames=varnames)
+        return code_obj
     else:
         return types.CodeType(*args)
 
 
-def instructions(code_obj):
+def instructions(code_obj, show_cache=True):
-    # easy for python 3.4+
+    if sys.version_info >= (3, 11):
-    if sys.version_info >= (3, 4):
+        # Python 3.11 introduced "cache instructions", hidden by default.
+        for inst in dis.Bytecode(code_obj, show_caches=show_cache):
+            yield inst
+    elif sys.version_info >= (3, 4): # easy for python 3.4+
         for inst in dis.Bytecode(code_obj):
             yield inst
     else:
@@ -171,7 +201,7 @@ def write_instruction(op, arg):
                 chr((arg >> 8) & 255)
             ]
         else:
-            raise ValueError("Invalid oparg: {0} is too large".format(oparg))
+            raise ValueError("Invalid oparg: {0} is too large".format(arg))
     else:  # python 3.6+ uses wordcode instead of bytecode and they already supply all the EXTENDEND_ARG ops :)
         if arg is None:
             return [chr(op), 0]
@@ -191,6 +221,7 @@ def write_instruction(op, arg):
         #     raise ValueError("Invalid oparg: {0} is too large".format(oparg))
 
 
+
 def check(code_obj):
     old_bytecode = code_obj.co_code
     insts = list(instructions(code_obj))
@@ -221,24 +252,99 @@ def check(code_obj):
                 'Your python version made changes to the bytecode')
 
 
+
+
+def signature(func):
+    code_obj = six.get_function_code(func)
+    return (code_obj.co_nlocals, code_obj.co_argcount, code_obj.co_nlocals, code_obj.co_stacksize,
+    code_obj.co_flags, code_obj.co_names, code_obj.co_varnames,
+    code_obj.co_filename,
+    code_obj.co_freevars, code_obj.co_cellvars)
+
 check(six.get_function_code(check))
 
+
+
+def compare_func(fake_func, gt_func):
+    print(signature(fake_func))
+    print(signature(gt_func))
+    assert signature(fake_func) == signature(gt_func)
+    fake_ins = list(instructions(six.get_function_code(fake_func), show_cache=False))
+    real_ins = list(instructions(six.get_function_code(gt_func), show_cache=False))
+    offset = 0
+    pos = 0
+    for e in fake_ins:
+        if e.opname == "NOP":
+            offset += 1  # ignore NOPs that are inserted in place of old cache.
+        else:
+            real = real_ins[pos]
+            fake = e
+            print("POS %d OFFSET: %d   FAKE VS REAL" % (pos, offset))
+            print(fake)
+            print(real)
+            assert fake.opcode == real.opcode
+            if fake.opcode in dis.hasjabs or fake.opcode in dis.hasjrel:
+                pass
+            else:
+                assert fake.arg == real.arg
+                assert fake.argval == real.argval or fake.opname in ["LOAD_CONST"]
+                assert fake.is_jump_target == real.is_jump_target
+
+            pos += 1
+    assert pos == len(real_ins), (pos, len(real_ins))
+    print("DONE, looks good.")
+
+
 if __name__ == '__main__':
-    x = 'Wrong'
+    import faulthandler
-    dick = 3000
 
-    def func(a):
+    faulthandler.enable()
-        print(x, y, z, a)
-        print(dick)
-        d = (x, )
-        for e in (e for e in x):
-            print(e)
-        return x, y, z
 
-    func2 = types.FunctionType(
+    def func(cmpfn):
-        append_arguments(six.get_function_code(func), ('x', 'y', 'z')),
+        if not this.Class in ('Array', 'Arguments'):
-        six.get_function_globals(func),
+            return this.to_object()  # do nothing
-        func.__name__,
+        arr = []
-        closure=six.get_function_closure(func))
+        for i in xrange(len(this)):
-    args = (2, 2, 3, 4), 3, 4
+            arr.append(this.get(six.text_type(i)))
-    assert func2(1, *args) == args
+
+        if not arr:
+            return this
+        if not cmpfn.is_callable():
+            cmpfn = None
+        cmp = lambda a, b: sort_compare(a, b, cmpfn)
+        if six.PY3:
+            key = functools.cmp_to_key(cmp)
+            arr.sort(key=key)
+        else:
+            arr.sort(cmp=cmp)
+        for i in xrange(len(arr)):
+            this.put(six.text_type(i), arr[i])
+
+        return this
+
+
+    def func_gt(cmpfn, this, arguments):
+        if not this.Class in ('Array', 'Arguments'):
+            return this.to_object()  # do nothing
+        arr = []
+        for i in xrange(len(this)):
+            arr.append(this.get(six.text_type(i)))
+
+        if not arr:
+            return this
+        if not cmpfn.is_callable():
+            cmpfn = None
+        cmp = lambda a, b: sort_compare(a, b, cmpfn)
+        if six.PY3:
+            key = functools.cmp_to_key(cmp)
+            arr.sort(key=key)
+        else:
+            arr.sort(cmp=cmp)
+        for i in xrange(len(arr)):
+            this.put(six.text_type(i), arr[i])
+
+        return this
+
+
+    func2 = fix_js_args(func)
+    compare_func(func2, func_gt)