Skip to content

Commit a976d6a

Browse files
thautwarmlfkdsk
thautwarm
authored and
lfkdsk
committed
fix try-stmt emitting + remove redundant analysis + able to run real world packages (#54)
* fix and simplify try stmt emitting * fix try-stmt and load closure * import sklearn and disp model * use staging python to extremely speed up parsing * fix import * & import as * bytecode patch * update req * use standard classdef impl & able to run sklearn
1 parent 97ab6d8 commit a976d6a

14 files changed

+155
-134
lines changed

‎import_invocation_test.py

+16-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,18 @@
1-
import yapypy.extended_python.pycompat as pycompat
1+
# use yapypy run
2+
from yapypy.extended_python import pycompat
23
pycompat.is_debug = True
3-
import platform
4+
from sklearn.datasets import load_iris
5+
from sklearn.ensemble import RandomForestClassifier
6+
from sklearn.model_selection import KFold
7+
from sklearn.metrics import classification_report
48

9+
rfc = RandomForestClassifier()
10+
data = load_iris()
11+
X = data.data
12+
y = data.target
13+
14+
kf = KFold(5, shuffle=True)
15+
for tr_idx, te_idx in kf.split(X):
16+
x1, x2, y1, y2 = X[tr_idx], X[te_idx], y[tr_idx], y[te_idx]
17+
rfc.fit(x1, y1)
18+
print(classification_report(y2, rfc.predict(x2)))

‎requirements.txt

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
yapf
22
pytest
33
pytest-cov
4-
bytecode==0.7.0
5-
Redy
4+
bytecode>=0.7.0
5+
Redy>=0.2.9
66
rbnf>=0.3.21
77
wisepy
88
flake8

‎setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
packages=['yapypy', 'yapypy.cmd', 'yapypy.extended_python', 'yapypy.utils', 'yapypy.extended_python.emit_impl'],
2121
entry_points={'console_scripts': ['yapypy=yapypy.cmd.cli:python_ex_cli']},
2222
install_requires=[
23-
'Redy', 'rbnf>=0.3.21', 'wisepy', 'bytecode==0.7.0', 'yapf',
23+
'Redy>=0.2.9', 'rbnf>=0.3.21', 'wisepy', 'bytecode>=0.7.0', 'yapf',
2424
'astpretty'
2525
],
2626
platforms='any',

‎snippet.py

+12-18
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def dis_code(code: types.CodeType, f):
7676

7777
def case(code, ctx, debug=False):
7878
stmt = parse(code).result
79-
code_obj = py_compile(stmt, is_entrypoint=True)
79+
code_obj = py_compile(stmt, is_entrypoint=False)
8080

8181
if debug:
8282
code_obj2 = compile(code, "", "exec")
@@ -101,23 +101,17 @@ def case(code, ctx, debug=False):
101101

102102
case(
103103
"""
104-
class S:
105-
pass
106-
print(S)
107-
108-
109-
class T(type):
110-
111-
def __new__(mcs, name, bases, ns):
112-
print(mcs, name, bases, ns)
113-
return type(name, bases, ns)
114-
115-
class S(list, metaclass=T):
116-
def get2(self):
117-
return self[2]
118-
119-
s = S([1, 2, 3])
120-
print(s.get2())
104+
def f():
105+
z = 1
106+
class S(dict):
107+
a = 1
108+
b = 2
109+
c = 3
110+
d = z
111+
def __init__(self):
112+
super().__init__()
113+
print(S)
114+
f()
121115
""",
122116
ctx,
123117
debug=True)

‎yapypy/cmd/cli.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,10 @@
33
import struct
44
import time
55
from importlib._bootstrap_external import MAGIC_NUMBER
6-
76
from Redy.Tools.PathLib import Path
87
from rbnf.edsl.rbnf_analyze import check_parsing_complete
98
from wisepy.talking import Talking
10-
9+
from yapypy.extended_python import pycompat
1110
from yapypy.extended_python.parser import parse as parse_ext_py
1211
from yapypy.extended_python.py_compile import py_compile
1312

‎yapypy/extended_python/emit_impl/exception_handling.py

+6-10
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ def py_emit(node: ast.Try, ctx: Context):
105105
>>> except TypeError as e:
106106
>>> a = 'type'
107107
>>> assert a == 'type'
108+
>>> print('exc handling done')
108109
"""
109110
lineno = node.lineno
110111
bodys = node.body
@@ -130,7 +131,7 @@ def py_emit(node: ast.Try, ctx: Context):
130131
byte_codes.append(POP_BLOCK())
131132
byte_codes.append(JUMP_FORWARD(try_forward))
132133
byte_codes.append(setup_forward)
133-
labels = [Label()] * (len(handlers) - 1)
134+
labels = [endfinally_forward, *(Label() for i in range(len(handlers) - 1))]
134135

135136
for (idx, handler) in enumerate(handlers):
136137
h_lineno = handler.lineno
@@ -139,18 +140,13 @@ def py_emit(node: ast.Try, ctx: Context):
139140
h_bodys = handler.body
140141

141142
if typ is not None:
142-
if idx > 0:
143-
dur_top = labels.pop()
144-
byte_codes.append(dur_top)
145143
byte_codes.append(DUP_TOP())
146144
py_emit(typ, ctx)
147145
byte_codes.append(COMPARE_OP(Compare.EXC_MATCH, lineno=h_lineno))
148-
if labels:
149-
byte_codes.append(POP_JUMP_IF_FALSE(labels[-1]))
150-
else:
151-
byte_codes.append(POP_JUMP_IF_FALSE(endfinally_forward))
146+
byte_codes.append(POP_JUMP_IF_FALSE(labels[-1]))
152147

153148
byte_codes.append(POP_TOP(lineno=h_lineno))
149+
154150
if name is not None:
155151
name_forward = Label()
156152
ctx.store_name(name)
@@ -162,7 +158,7 @@ def py_emit(node: ast.Try, ctx: Context):
162158

163159
for hbody in h_bodys:
164160
py_emit(hbody, ctx)
165-
# byte_codes.append( POP_EXCEPT( ) )
161+
166162
if name is not None:
167163
byte_codes.append(POP_BLOCK())
168164
byte_codes.append(LOAD_CONST(None))
@@ -174,8 +170,8 @@ def py_emit(node: ast.Try, ctx: Context):
174170

175171
byte_codes.append(POP_EXCEPT())
176172
byte_codes.append(JUMP_FORWARD(except_forward))
173+
byte_codes.append(labels.pop())
177174

178-
byte_codes.append(endfinally_forward)
179175
byte_codes.append(END_FINALLY())
180176
byte_codes.append(try_forward)
181177

‎yapypy/extended_python/emit_impl/imports.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def py_emit(node: ast.Import, ctx: Context):
2525
),
2626
), # TOS for fromlist()
2727
byte_code.append(Instr("IMPORT_NAME", name.name, lineno=node.lineno))
28-
as_name = name.name or name.asname
28+
as_name = name.asname or name.name
2929
ctx.store_name(as_name, lineno=node.lineno)
3030

3131

@@ -35,12 +35,12 @@ def py_emit(node: ast.ImportFrom, ctx: Context):
3535
title: import from
3636
test:
3737
>>> from os.path import join
38-
>>> from os import path as path
38+
>>> from os import path as _path
3939
>>> from os import *
4040
>>> from os.path import *
4141
>>> def f(x):
4242
>>> x
43-
>>>
43+
>>> print(_path)
4444
>>> print(join('a', 'b'))
4545
>>> print(f(1))
4646
>>> x, y = 1, 2
@@ -59,6 +59,6 @@ def py_emit(node: ast.ImportFrom, ctx: Context):
5959
else:
6060
for name in node.names:
6161
ctx.bc.append(Instr("IMPORT_FROM", name.name, lineno=lineno))
62-
as_name = name.name or name.asname
62+
as_name = name.asname or name.name
6363
ctx.store_name(as_name, lineno=lineno)
6464
ctx.bc.append(POP_TOP(lineno=lineno))

‎yapypy/extended_python/emit_impl/new_context.py

+22-46
Original file line numberDiff line numberDiff line change
@@ -107,12 +107,7 @@ def emit_function(node: typing.Union[ast.AsyncFunctionDef, ast.FunctionDef, ast.
107107
new_ctx.bc.append(Instr('LOAD_CONST', None))
108108
new_ctx.bc.append(Instr('RETURN_VALUE'))
109109

110-
try:
111-
inner_code = new_ctx.bc.to_code()
112-
except RuntimeError:
113-
print(new_ctx.bc.filename)
114-
dump_bytecode(new_ctx.bc)
115-
raise
110+
inner_code = new_ctx.bc.to_code()
116111
parent_ctx.bc.append(Instr('LOAD_CONST', inner_code, lineno=node.lineno))
117112

118113
# when it comes to nested, the name is not generated correctly now.
@@ -195,64 +190,45 @@ def py_emit(node: ast.ClassDef, ctx: Context):
195190
name = node.name
196191
parent_ctx: Context = ctx.parent
197192

198-
ctx.bc.name = f'{parent_ctx.bc.name}.{name}' if parent_ctx.bc.name else name
199-
200193
for decorator in getattr(node, 'decorator_list', ()):
201194
py_emit(decorator, parent_ctx)
202195

196+
parent_ctx.bc.append(Instr('LOAD_BUILD_CLASS'))
197+
ctx.bc.name = f'{parent_ctx.bc.name}.{name}' if parent_ctx.bc.name else name
198+
203199
head = node.body
204200
if isinstance(head, ast.Expr) and isinstance(head.value, ast.Str):
205201
ctx.bc.docstring = head.value.s
206202

207-
for each in node.body:
208-
py_emit(each, ctx)
209-
210203
ctx.bc.argcount = 0
211204
ctx.bc.kwonlyarbgcount = 0
212-
ctx.bc.argnames = ['.yapypy.args', '.yapypy.kwargs']
213-
214-
make_function_flags = 0
215-
ctx.bc.flags |= CompilerFlags.VARARGS
216-
ctx.bc.flags |= CompilerFlags.VARKEYWORDS
217-
218-
if ctx.sym_tb.freevars:
219-
make_function_flags |= 0x08
220-
ctx.load_closure(lineno=node.lineno)
221-
205+
ctx.load_closure(lineno=node.lineno)
222206
ctx.bc.extend([
223207
LOAD_GLOBAL('__name__'),
224-
STORE_FAST('__module__'),
208+
STORE_NAME('__module__'),
225209
LOAD_CONST(ctx.bc.name),
226-
STORE_FAST('__qualname__'),
227-
LOAD_FAST('.yapypy.kwargs'),
228-
LOAD_ATTR('get'),
229-
LOAD_CONST('metaclass'),
230-
LOAD_GLOBAL('.yapypy.type'),
231-
CALL_FUNCTION(2), # get metaclass
232-
LOAD_CONST(name),
233-
LOAD_FAST('.yapypy.args'),
234-
LOAD_GLOBAL('.yapypy.locals'),
235-
CALL_FUNCTION(0), # get locals
236-
DUP_TOP(),
237-
LOAD_ATTR('pop'),
238-
DUP_TOP(),
239-
LOAD_CONST('.yapypy.args'),
240-
CALL_FUNCTION(1),
241-
POP_TOP(),
242-
LOAD_CONST('.yapypy.kwargs'),
243-
CALL_FUNCTION(1),
244-
POP_TOP(),
245-
CALL_FUNCTION(3, lineno=node.lineno), # create new type
210+
STORE_NAME('__qualname__'),
211+
])
212+
for each in node.body:
213+
py_emit(each, ctx)
214+
215+
# https://docs.python.org/3/reference/datamodel.html#creating-the-class-object
216+
ctx.bc.extend([
217+
Instr('LOAD_CLOSURE', CellVar('__class__')),
218+
STORE_NAME('__classcell__'),
219+
LOAD_CONST(None),
220+
RETURN_VALUE()
246221
])
247222

248-
ctx.bc.append(Instr('RETURN_VALUE'))
249223
inner_code = ctx.bc.to_code()
250224

251225
parent_ctx.bc.append(LOAD_CONST(inner_code, lineno=lineno))
252226
# when it comes to nested, the name is not generated correctly now.
253227
parent_ctx.bc.append(LOAD_CONST(name, lineno=lineno))
254228

255-
parent_ctx.bc.append(MAKE_FUNCTION(make_function_flags, lineno=lineno))
229+
parent_ctx.bc.append(MAKE_FUNCTION(0x08, lineno=lineno))
230+
231+
parent_ctx.bc.extend([LOAD_CONST(name), BUILD_TUPLE(2)])
256232

257233
# *args
258234
if node.bases:
@@ -262,6 +238,8 @@ def py_emit(node: ast.ClassDef, ctx: Context):
262238
else:
263239
parent_ctx.bc.append(LOAD_CONST(()))
264240

241+
parent_ctx.bc.append(Instr('BUILD_TUPLE_UNPACK_WITH_CALL', 2))
242+
265243
# **kwargs
266244
if node.keywords:
267245
keys, values = zip(*[(ast.Str(
@@ -274,9 +252,7 @@ def py_emit(node: ast.ClassDef, ctx: Context):
274252
py_emit(ex_dict, parent_ctx)
275253
else:
276254
parent_ctx.bc.append(BUILD_MAP(0))
277-
278255
parent_ctx.bc.append(CALL_FUNCTION_EX(1))
279-
280256
parent_ctx.bc.extend(
281257
[CALL_FUNCTION(1, lineno=lineno)] * len(getattr(node, 'decorator_list', ())))
282258

‎yapypy/extended_python/parser.py

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ def lex(text: t.Union[str, bytes]):
4242
python.namespace.update({**extended_ast.__dict__, **helper.__dict__, **ast.__dict__})
4343
build_language(RBNF, python, '<grammar>')
4444
python_parser = python.named_parsers['file_input']
45+
python.as_fixed()
4546

4647

4748
def _find_error(source_code, tokens, state):

‎yapypy/extended_python/py_compile.py

+6-8
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,17 @@ def py_compile(node, filename='<unknown>', is_entrypoint=False):
1010
ctx = _non_ctx.enter_new(node.tag)
1111
ctx.bc.filename = filename
1212
ctx.bc.name = '__main__' if is_entrypoint else splitext(
13-
Path(filename).relative())[1]
14-
15-
ctx.bc.append(LOAD_GLOBAL('type'))
16-
ctx.bc.append(STORE_GLOBAL('.yapypy.type'))
17-
ctx.bc.append(LOAD_GLOBAL('locals'))
18-
ctx.bc.append(STORE_GLOBAL('.yapypy.locals'))
19-
13+
Path(filename).relative())[0]
2014
try:
2115
py_emit(node.it, ctx)
2216
except SyntaxError as exc:
2317
exc.filename = filename
2418
raise exc
25-
return ctx.bc.to_code()
19+
try:
20+
return ctx.bc.to_code()
21+
except Exception as e:
22+
dump_bytecode(ctx.bc)
23+
raise e
2624
else:
2725
tag = to_tagged_ast(node)
2826
return py_compile(tag, filename, is_entrypoint=is_entrypoint)

0 commit comments

Comments
 (0)