diff --git a/dcrpcgen/main.py b/dcrpcgen/main.py index 46bea34..2416aee 100644 --- a/dcrpcgen/main.py +++ b/dcrpcgen/main.py @@ -8,6 +8,7 @@ from ._version import __version__ from .go import add_go_cmd from .java import add_java_cmd +from .python import add_python_cmd def get_schema(filename: str) -> dict: @@ -48,7 +49,7 @@ def get_parser() -> argparse.ArgumentParser: parser.add_argument("-v", "--version", action="version", version=__version__) subparsers = parser.add_subparsers(title="subcommands") - for add_generator in [add_java_cmd, add_go_cmd]: + for add_generator in [add_java_cmd, add_go_cmd, add_python_cmd]: add_generator(subparsers, base) return parser diff --git a/dcrpcgen/python/__init__.py b/dcrpcgen/python/__init__.py new file mode 100644 index 0000000..da8c860 --- /dev/null +++ b/dcrpcgen/python/__init__.py @@ -0,0 +1,106 @@ +"""Python code generation""" + +from argparse import Namespace +from pathlib import Path +from typing import Any + +from ..utils import camel2snake +from .templates import get_template +from .types import decode_type, generate_types +from .utils import create_comment + + +def python_cmd(args: Namespace) -> None: + """Generate JSON-RPC client for the Python programming language""" + root_folder = Path(args.folder) + root_folder.mkdir(parents=True, exist_ok=True) + + path = root_folder / "types.py" + print(f"Generating {path}") + generate_types(path, args.openrpc_spec["components"]["schemas"]) + + path = root_folder / "rpc.py" + print(f"Generating {path}") + generate_methods(path, args.openrpc_spec["methods"]) + + path = root_folder / "transport.py" + print(f"Generating {path}") + generate_transport(path) + + path = root_folder / "_utils.py" + print(f"Generating {path}") + generate_utils(path) + + +def generate_methods(path: Path, methods: dict[str, Any]) -> str: + """Generate Rpc class""" + with path.open("w", encoding="utf-8") as output: + template = get_template("rpc.py.j2") + output.write( + template.render( + methods=methods, + generate_method=generate_method, + ) + ) + + +def generate_method(method: dict[str, Any]) -> str: + """Generate a rpc module""" + assert method["paramStructure"] == "by-position" + params = method["params"] + params_types = {param["name"]: decode_type(param["schema"]) for param in params} + result_type = decode_type(method["result"]["schema"]) + name = method["name"] + tab = " " + text = "" + text += f"{tab}def {name}(" + text += ", ".join( + f'{camel2snake(param["name"])}: {params_types[param["name"]]}' + for param in params + ) + text += f") -> {result_type}:\n" + if "description" in method: + text += create_comment(method["description"], tab * 2, docstr=True) + + args = [f'"{name}"'] + for param in params: + param_name = camel2snake(param["name"]) + if params_types[param["name"]] in ( + "bool", + "int", + "float", + "str", + "Optional[bool]", + "Optional[int]", + "Optional[float]", + "Optional[str]", + "list[bool]", + "list[int]", + "list[float]", + "list[str]", + "dict[Any, Optional[str]]", + "dict[Any, str]", + "tuple[float, float]", + ): + args.append(param_name) + else: + args.append(f"_wrap({param_name})") + + rtn = "" if result_type == "None" else "return " + stmt = f'transport.call({", ".join(args)})' + text += f"{tab*2}{rtn}{stmt}\n" + return text + + +def generate_transport(path: Path) -> str: + """Generate transport module""" + with path.open("w", encoding="utf-8") as output: + template = get_template("transport.py.j2") + output.write(template.render()) + + +def generate_utils(path: Path) -> str: + """Generate utils module""" + with path.open("w", encoding="utf-8") as output: + template = get_template("utils.py.j2") + output.write(template.render()) diff --git a/dcrpcgen/python/templates/NormalClass.py.j2 b/dcrpcgen/python/templates/NormalClass.py.j2 new file mode 100644 index 0000000..07c20e8 --- /dev/null +++ b/dcrpcgen/python/templates/NormalClass.py.j2 @@ -0,0 +1,4 @@ +@dataclass(kw_only=True) +class {{ name }}: +{% if "description" in schema %}{{ create_comment(schema["description"], " ", docstr=True) + "\n" }}{% endif -%} +{{ generate_properties(schema["properties"], False) -}} diff --git a/dcrpcgen/python/templates/StrEnumTemplate.py.j2 b/dcrpcgen/python/templates/StrEnumTemplate.py.j2 new file mode 100644 index 0000000..0248b1d --- /dev/null +++ b/dcrpcgen/python/templates/StrEnumTemplate.py.j2 @@ -0,0 +1,10 @@ +class {{ name }}(StrEnum): +{% for schema in schemas -%} + {% if "description" in schema %} + {{- create_comment(schema["description"], " ").rstrip() }} + {%- endif %} + {%- for val in schema["enum"] %} + {{ camel2snake(val) | upper }} = "{{ val }}" + {%- endfor %} + +{% endfor -%} diff --git a/dcrpcgen/python/templates/UnionType.py.j2 b/dcrpcgen/python/templates/UnionType.py.j2 new file mode 100644 index 0000000..8c7d7be --- /dev/null +++ b/dcrpcgen/python/templates/UnionType.py.j2 @@ -0,0 +1,5 @@ +{% for typ in schema["oneOf"] %} + {{- generate_subtype(typ, name) }} +{% endfor -%} +{% if "description" in schema %}{{ create_comment(schema["description"]) + "\n" }}{% endif -%} +{{ name }}: TypeAlias = {{ get_union_type(schema["oneOf"], name) }} diff --git a/dcrpcgen/python/templates/__init__.py b/dcrpcgen/python/templates/__init__.py new file mode 100644 index 0000000..cb97b4a --- /dev/null +++ b/dcrpcgen/python/templates/__init__.py @@ -0,0 +1,12 @@ +"""Code templates""" + +from jinja2 import Environment, PackageLoader, Template + +env = Environment( + loader=PackageLoader(__name__.rsplit(".", maxsplit=1)[0], "templates") +) + + +def get_template(name: str) -> Template: + """Load the template with the given filename""" + return env.get_template(name) diff --git a/dcrpcgen/python/templates/rpc.py.j2 b/dcrpcgen/python/templates/rpc.py.j2 new file mode 100644 index 0000000..eb6d30d --- /dev/null +++ b/dcrpcgen/python/templates/rpc.py.j2 @@ -0,0 +1,29 @@ +"""JSON-RPC API definition.""" + +import dataclasses +from typing import Any, Optional + +from .transport import RpcTransport +from .types import * + + +class Rpc: + """Access to the chatmail JSON-RPC API.""" + + def __init__(self, transport: RpcTransport) -> None: + self.transport = transport + +{% for method in methods %} + {{- generate_method(method)}} +{% endfor -%} + + +def _wrap(arg: Any) -> Any: + if isinstance(arg, dict): + return {key: _wrap(val) for key, val in arg.items()} + if isinstance(arg, list): + return [_wrap(elem) for elem in arg] + + if dataclasses.is_dataclass(arg): + return snake2camel(dataclasses.asdict(arg)) + return arg \ No newline at end of file diff --git a/dcrpcgen/python/templates/transport.py.j2 b/dcrpcgen/python/templates/transport.py.j2 new file mode 100644 index 0000000..d2abd74 --- /dev/null +++ b/dcrpcgen/python/templates/transport.py.j2 @@ -0,0 +1,153 @@ +"""JSON-RPC transports to communicate with chatmail core.""" + +import itertools +import json +import logging +import os +import subprocess +import sys +from abc import ABC, abstractmethod +from queue import Queue +from threading import Event, Thread +from typing import Any, Dict, Iterator, Optional + +from ._utils import to_attrdict + + +class JsonRpcError(Exception): + """An error occurred in your request to the JSON-RPC API.""" + + +class RpcTransport(ABC): + """Chatmail RPC client's transport.""" + + @abstractmethod + def call(self, method: str, *args) -> Any: + """Request the RPC server to call a function and return its return value if any.""" + + +class _Result(Event): + def __init__(self) -> None: + self._value: Any = None + super().__init__() + + def set(self, value: Any) -> None: # noqa + self._value = value + super().set() + + def wait(self) -> Any: # noqa + super().wait() + return self._value + + +class IOTransport: + """Chatmail RPC transport over IO using external deltachat-rpc-server program.""" + + def __init__(self, accounts_dir: Optional[str] = None, rpc_executable: str = "deltachat-rpc-server", **kwargs): + """The given arguments will be passed to subprocess.Popen()""" + self.logger = logging.getLogger("deltachat2.IOTransport") + if accounts_dir: + kwargs["env"] = { + **kwargs.get("env", os.environ), + "DC_ACCOUNTS_PATH": str(accounts_dir), + } + self.rpc_executable = rpc_executable + self._kwargs = kwargs + self.process: subprocess.Popen + self.id_iterator: Iterator[int] + # Map from request ID to the result. + self.pending_results: Dict[int, _Result] + self.request_queue: Queue + self.closing: bool + self.reader_thread: Thread + self.writer_thread: Thread + + def start(self) -> None: + """Start the RPC server process.""" + if sys.version_info >= (3, 11): + # Prevent subprocess from capturing SIGINT. + kwargs = {"process_group": 0, **self._kwargs} + else: + # `process_group` is not supported before Python 3.11. + kwargs = {"preexec_fn": os.setpgrp, **self._kwargs} # noqa: PLW1509 + self.process = subprocess.Popen( # noqa: R1732 + self.rpc_executable, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + **kwargs, + ) + self.id_iterator = itertools.count(start=1) + self.pending_results = {} + self.request_queue = Queue() + self.closing = False + self.reader_thread = Thread(target=self._reader_loop) + self.reader_thread.start() + self.writer_thread = Thread(target=self._writer_loop) + self.writer_thread.start() + + def close(self) -> None: + """Terminate RPC server process and wait until the reader loop finishes.""" + self.closing = True + self.call("stop_io_for_all_accounts") + assert self.process.stdin + self.process.stdin.close() + self.reader_thread.join() + self.request_queue.put(None) + self.writer_thread.join() + + def __enter__(self): + self.start() + return self + + def __exit__(self, _exc_type, _exc, _tb): + self.close() + + def _reader_loop(self) -> None: + try: + assert self.process.stdout + while True: + line = self.process.stdout.readline() + if not line: # EOF + break + response = json.loads(line) + if "id" in response: + self.pending_results.pop(response["id"]).set(response) + else: + self.logger.warning("Got a response without ID: %s", response) + except Exception: + # Log an exception if the reader loop dies. + self.logger.exception("Exception in the reader loop") + + def _writer_loop(self) -> None: + """Writer loop ensuring only a single thread writes requests.""" + try: + assert self.process.stdin + while True: + request = self.request_queue.get() + if not request: + break + data = (json.dumps(request) + "\n").encode() + self.process.stdin.write(data) + self.process.stdin.flush() + except Exception: + # Log an exception if the writer loop dies. + self.logger.exception("Exception in the writer loop") + + def call(self, method: str, *args) -> Any: + """Request the RPC server to call a function and return its return value if any.""" + request_id = next(self.id_iterator) + request = { + "jsonrpc": "2.0", + "method": method, + "params": args, + "id": request_id, + } + result = self.pending_results[request_id] = _Result() + self.request_queue.put(request) + response = result.wait() + + if "error" in response: + raise JsonRpcError(response["error"]) + if "result" in response: + return to_attrdict(response["result"]) + return None diff --git a/dcrpcgen/python/types.py b/dcrpcgen/python/types.py new file mode 100644 index 0000000..e2342cc --- /dev/null +++ b/dcrpcgen/python/types.py @@ -0,0 +1,178 @@ +"""Types generation""" + +from pathlib import Path +from typing import Any + +from ..utils import camel2snake +from .templates import get_template +from .utils import create_comment + + +def generate_strenum(name: str, schemas: list[dict]) -> str: + """Generate a StrEnum""" + print("Generating", name) + template = get_template("StrEnumTemplate.py.j2") + return template.render( + name=name, + schemas=schemas, + create_comment=create_comment, + camel2snake=camel2snake, + ) + + +def generate_uniontype(name: str, schema: dict[str, Any]) -> str: + """Generate union type""" + print("Generating", name) + template = get_template("UnionType.py.j2") + return template.render( + name=name, + schema=schema, + create_comment=create_comment, + get_union_type=get_union_type, + generate_subtype=generate_subtype, + ) + + +def get_union_type(schema: list[dict[str, Any]], parent: str) -> str: + """Get union type definition given list of child class schema""" + kind_names = [parent + get_subtype_name(typ) for typ in schema] + return " | ".join(kind_names) + + +def get_subtype_name(schema: dict[str, Any], capitalize=True) -> str: + """Get class name from the given child class schema""" + assert schema["type"] == "object" + kind = schema["properties"]["kind"] + assert kind["type"] == "string" + assert len(kind["enum"]) == 1 + name = kind["enum"][0] + return name[0].upper() + name[1:] if capitalize else name + + +def generate_subtype(schema: dict[str, Any], parent: str) -> str: + """Generate child inner class""" + name = parent + get_subtype_name(schema) + print(f" Generating {name}") + props = generate_properties(schema["properties"], True) + + if not props: + if desc := schema.get("description"): + doc = create_comment(desc) + else: + doc = "" + val = get_subtype_name(schema, False) + return f'{doc}{name} = "{val}"' + + text = "@dataclass(kw_only=True)\n" + text += f"class {name}:\n" + if desc := schema.get("description"): + text += create_comment(desc, " ", docstr=True) + text += props + text += "\n" + return text + + +def generate_class(name: str, schema: dict[str, Any]) -> str: + """Generate normal standalone class type (no child class, no super-class)""" + print("Generating", name) + template = get_template("NormalClass.py.j2") + return template.render( + name=name, + schema=schema, + create_comment=create_comment, + generate_properties=generate_properties, + ) + + +def generate_properties(properties: dict[str, Any], is_subclass: bool) -> str: + """Generate class fields""" + tab = " " + text = "" + for property_name, property_desc in properties.items(): + if is_subclass and property_name == "kind": + continue + property_name = camel2snake(property_name) + typ = decode_type(property_desc) + if desc := property_desc.get("description"): + text += "\n" + create_comment(desc, tab) + if mini := property_desc.get("minimum"): + minimum = create_comment(f"minimum value: {mini}", " ") + else: + minimum = "\n" + text += f"{tab}{property_name}: {typ}{minimum}" + return text + + +def decode_type(property_desc: dict[str, Any]) -> str: + """Decode a type, it can be a returning type or parameter type""" + schemas_url = "#/components/schemas/" + + if "anyOf" in property_desc: + assert len(property_desc["anyOf"]) == 2 + assert property_desc["anyOf"][1] == {"type": "null"} + ref = property_desc["anyOf"][0]["$ref"] + assert ref.startswith(schemas_url) + typ = ref.removeprefix(schemas_url) + return f"Optional[{typ}]" + + if "$ref" in property_desc: + assert property_desc["$ref"].startswith(schemas_url) + typ = property_desc["$ref"].removeprefix(schemas_url) + return typ + + if property_desc["type"] == "null": + return "None" # only for function returning type + + if "null" in property_desc["type"]: + assert len(property_desc["type"]) == 2 + assert property_desc["type"][1] == "null" + property_desc["type"] = property_desc["type"][0] + if typ := decode_type(property_desc): + return f"Optional[{typ}]" + elif property_desc["type"] == "boolean": + return "bool" + elif property_desc["type"] == "integer": + return "int" + elif property_desc["type"] == "number" and property_desc["format"] == "double": + return "float" + elif property_desc["type"] == "string": + return "str" + elif property_desc["type"] == "array": + if isinstance(property_desc["items"], list): + types = ", ".join(decode_type(x) for x in property_desc["items"]) + return f"tuple[{types}]" + + items_type = decode_type(property_desc["items"]) + return f"list[{items_type}]" + elif "additionalProperties" in property_desc: + additional_properties = property_desc["additionalProperties"] + return f"dict[Any, {decode_type(additional_properties)}]" + + raise ValueError(f"Not supported: {property_desc!r}") + + +def generate_types(path: Path, schemas: dict[str, Any]) -> None: + """Generate classes and enumerations from RPC type definitions""" + items = [] + for name, schema in sorted(schemas.items(), key=lambda e: e[0]): + if "oneOf" in schema: + if all(typ["type"] == "string" for typ in schema["oneOf"]): + # Simple enumeration consisting only of various string types. + items.append(generate_strenum(name, schema["oneOf"])) + else: + # Union type. + items.append(generate_uniontype(name, schema)) + elif schema["type"] == "string": + items.append(generate_strenum(name, [schema])) + elif schema["type"] == "object": + items.append(generate_class(name, schema)) + else: + raise ValueError(f"Unknow schema: {schema}") + + with path.open("w", encoding="utf-8") as output: + output.write(f'"""Data classes and types from the JSON-RPC."""\n\n') + output.write("from dataclasses import dataclass\n") + output.write("from enum import StrEnum\n") + output.write("from typing import Any, Optional, TypeAlias\n") + output.write("\n\n") + output.write("\n\n".join(items)) diff --git a/dcrpcgen/python/utils.py b/dcrpcgen/python/utils.py new file mode 100644 index 0000000..d46dd16 --- /dev/null +++ b/dcrpcgen/python/utils.py @@ -0,0 +1,21 @@ +"""Utilities for Python code generation.""" + + +def create_comment(text: str, indentation: str = "", docstr: bool = False) -> str: + """Generate a Python comment""" + if docstr: + if "\n" not in text: + return f'{indentation}"""{text.strip()}"""\n' + + comment = f'{indentation}"""\n' + for line in text.split("\n"): + comment += f"{indentation}{line.strip()}\n" + return comment + f'{indentation}"""\n' + + if "\n" not in text: + return f"{indentation}# {text.strip()}\n" + + comment = "" + for line in text.split("\n"): + comment += f"{indentation}# {line.strip()}\n" + return comment