From 229d25394ab309396be250fe7e194a5e0fc72ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Sat, 31 Oct 2026 18:46:30 +0100 Subject: [PATCH 1/2] Call a writer with the out-parameters its catalog entry states CtypesEngine.encode calls a writer with its own signature: every out-parameter the catalog lists in the writer's shape.outParams gets a slot of its own at its position, the trailing inputs fill the other positions in order, and a writer handing back its size in a size_t * is read for that many bytes, less the terminating NUL MEOS counts in the size of a string. The server reads each writer's out-parameters from the catalog (writer_outparams) and states them in the route as encode_outparams, beside encode_aux, for a result, an out-parameter result and the elements of an array result. set_as_hexwkb, span_as_hexwkb, spanset_as_hexwkb and raster_as_hexwkb, the writers of 459 results on the wire over the catalog of MobilityDB master 1ff2270f7f, are called with their size_t *size_out; no reader on the wire has an out-parameter. Witness: tests/test_engine_integration.py, an intset, an intspan and a tint written by set_as_hexwkb, span_as_hexwkb and temporal_as_hexwkb through CtypesEngine.encode and read back to their value, and tests/test_server.py, the route of a result whose writer states size_out and the call the engine receives; both fail on master's server. Why: the encode a route names for a Set, a Span, a SpanSet or a Raster called the hex writer with its value and its variant and no size_out, leaving MEOS to write the size through whatever the third argument register held. Measured: the 632 tests pass, master's 630 and the 2 above, with libmeos built from MobilityDB master 1ff2270f7f. --- server/app.py | 34 ++++++++++++++++++--- server/engine.py | 52 ++++++++++++++++++++++++++------ tests/test_engine_integration.py | 22 ++++++++++++++ tests/test_server.py | 36 +++++++++++++++++++++- 4 files changed, 130 insertions(+), 14 deletions(-) diff --git a/server/app.py b/server/app.py index 0350959..b1c42cd 100644 --- a/server/app.py +++ b/server/app.py @@ -39,9 +39,31 @@ def build_enum_map(catalog: dict) -> dict: } +def writer_outparams(catalog: dict) -> dict: + """{writer: [(position, cType)]}: the out-parameters each function states in its + ``shape.outParams``, by position, the value at 0 (``set_as_hexwkb``: + ``[(2, "size_t *")]``), which #CtypesEngine.encode of server/engine.py fills.""" + out = {} + for fn in catalog.get("functions", []): + names = set((fn.get("shape") or {}).get("outParams") or ()) + if names: + out[fn["name"]] = [(i, p["cType"]) for i, p in enumerate(fn["params"]) + if p["name"] in names] + return out + + +def _with_outparams(value: dict | None, outs: dict) -> dict | None: + """``value`` (a wire result or element) with ``encode_outparams``, the + out-parameters of the writer it names.""" + if not value or not value.get("encode"): + return value + return {**value, "encode_outparams": outs.get(value["encode"], [])} + + def build_routes(catalog: dict) -> dict: """`/{fn}` → route descriptor, for every exposable function.""" routes = {} + outs = writer_outparams(catalog) for fn in catalog.get("functions", []): if not fn.get("network", {}).get("exposable"): continue @@ -70,10 +92,11 @@ def build_routes(catalog: dict) -> dict: "result": {"kind": r["kind"], "json": r.get("json"), "encode": r.get("encode"), "encode_aux": r.get("encode_aux", []), + "encode_outparams": outs.get(r.get("encode"), []), "from_outparam": r.get("from_outparam"), "out_ctype": r.get("out_ctype"), "presence_return": r.get("presence_return", False), - "element": r.get("element"), + "element": _with_outparams(r.get("element"), outs), "count_outparam": r.get("count_outparam"), "count_param": r.get("count_param")}, } @@ -165,7 +188,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict): return None if res["kind"] == "serialized": # opaque out-param -> encode return {"result": engine.encode( - res["encode"], val, _aux_args(res.get("encode_aux", [])))} + res["encode"], val, _aux_args(res.get("encode_aux", [])), + res.get("encode_outparams", []))} return {"result": bool(val) if res.get("json") == "boolean" else val} if res["kind"] == "array": @@ -181,7 +205,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict): # Elem **f(.., int *count): MEOS returns a fresh array + byref count. ptrs = engine.invoke_array(route["name"], args) return {"result": [engine.encode(el["encode"], p, - _aux_args(el.get("encode_aux", []))) + _aux_args(el.get("encode_aux", [])), + el.get("encode_outparams", [])) for p in ptrs]} rtag = "void" if res["kind"] == "void" else ( @@ -193,7 +218,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict): return None if res["kind"] == "serialized": return {"result": engine.encode( - res["encode"], out, _aux_args(res.get("encode_aux", [])))} + res["encode"], out, _aux_args(res.get("encode_aux", [])), + res.get("encode_outparams", []))} if res["kind"] == "json" and res["json"] == "boolean": return {"result": bool(out)} return {"result": out} diff --git a/server/engine.py b/server/engine.py index 7bd6323..b511831 100644 --- a/server/engine.py +++ b/server/engine.py @@ -22,7 +22,7 @@ decode(fn_name, value:str) -> handle (opaque) invoke(fn_name, args:[(tag,val)], rt) -> handle|scalar|None - encode(fn_name, handle) -> str + encode(fn_name, handle, aux, outparams) -> str """ from __future__ import annotations @@ -46,7 +46,9 @@ def decode(self, fn_name: str, value: str, aux=()): def invoke(self, fn_name: str, args: list, result_tag: str): raise NotImplementedError - def encode(self, fn_name: str, handle, aux=()) -> str: + def encode(self, fn_name: str, handle, aux=(), outparams=()) -> str: + """``outparams``: the ``(position, cType)`` of the writer's out-parameters, + from the catalog's ``shape.outParams`` (#writer_outparams of server/app.py).""" raise NotImplementedError def invoke_outparam(self, fn_name: str, args: list, out_ctype: str, @@ -83,7 +85,7 @@ def invoke(self, fn_name, args, result_tag): return {"str": "", "int": 0, "double": 0.0, "bool": False, "enum": ""}.get(result_tag, "") - def encode(self, fn_name, handle, aux=()): + def encode(self, fn_name, handle, aux=(), outparams=()): return f"" def invoke_outparam(self, fn_name, args, out_ctype, presence): @@ -198,15 +200,47 @@ def invoke(self, fn_name, args, result_tag): self._raise_if_error() return r - def encode(self, fn_name, handle, aux=()): + def encode(self, fn_name, handle, aux=(), outparams=()): + """``char *f(const T *, aux..., size_t *size_out)``: the writer called with its + own signature. ``outparams`` are the ``(position, cType)`` of the parameters + the catalog states in its ``shape.outParams``, positions counted from the value + at 0; each gets a slot of its own, as #invoke_outparam gives its result one, + and the trailing inputs fill the other positions in order. A writer handing + back its size in a ``size_t *`` is read for that many bytes.""" self._last_error = None f = self._sym(fn_name) + ct = self._ct atypes, avals = self._aux_ctypes(aux) - f.argtypes = [self._ct.c_void_p, *atypes] - f.restype = self._ct.c_char_p - r = f(handle, *avals) + outs = dict(outparams) + argtypes, cargs, slots = [ct.c_void_p], [handle], {} + rest = iter(zip(atypes, avals)) + for pos in range(1, 1 + len(atypes) + len(outs)): + if pos in outs: + slot = self._pointee_ctype(outs[pos])() + slots[pos] = (outs[pos], slot) + argtypes.append(ct.POINTER(type(slot))) + cargs.append(ct.byref(slot)) + else: + t, v = next(rest) + argtypes.append(t) + cargs.append(v) + f.argtypes = argtypes + if not slots: + f.restype = ct.c_char_p + r = f(*cargs) + self._raise_if_error() + return r.decode() if r else None + f.restype = ct.c_void_p + r = f(*cargs) self._raise_if_error() - return r.decode() if r else None + if not r: + return None + sizes = [s.value for c, s in slots.values() + if " ".join(c.replace("*", " ").split()) == "size_t"] + raw = ct.string_at(r, sizes[0]) if sizes else ct.string_at(r) + # The size of a string MEOS writes counts its terminating NUL + # (``datum_as_wkb`` sizes a hex buffer ``2 * size + 1``). + return (raw[:-1] if raw.endswith(b"\0") else raw).decode() def _pointee_ctype(self, out_ctype: str): ct = self._ct @@ -217,7 +251,7 @@ def _pointee_ctype(self, out_ctype: str): "long long": ct.c_longlong, "unsigned long": ct.c_ulong, "long": ct.c_long, "unsigned int": ct.c_uint, "short": ct.c_short, "unsigned char": ct.c_ubyte, - "signed char": ct.c_byte, + "signed char": ct.c_byte, "size_t": ct.c_size_t, }.get(base, ct.c_int) def invoke_outparam(self, fn_name, args, out_ctype, presence): diff --git a/tests/test_engine_integration.py b/tests/test_engine_integration.py index 6b40efc..7dc39e2 100644 --- a/tests/test_engine_integration.py +++ b/tests/test_engine_integration.py @@ -179,6 +179,28 @@ def test_the_service_wire_keeps_the_srid(self): [("ptr", self.eng.decode(te["in"], plain))], "int") self.assertEqual(srid0, 0) + def test_each_hex_writer_round_trips_through_encode(self): + """A Set, a Span and a Temporal written by their hex-WKB writer through + #CtypesEngine.encode, with the out-parameters the catalog states + (#writer_outparams of server/app.py), read back to the value, as + #test_each_type_round_trips_through_its_own_writer round-trips the text.""" + from server.app import writer_outparams + outs = writer_outparams(json.loads(_CATALOG.read_text())) + for reader, literal, writer, back, out in ( + ("intset_in", "{1, 2, 3}", "set_as_hexwkb", "set_from_hexwkb", + "intset_out"), + ("intspan_in", "[1, 5)", "span_as_hexwkb", "span_from_hexwkb", + "intspan_out"), + ("tint_in", "{1@2000-01-01, 2@2000-01-02}", "temporal_as_hexwkb", + "temporal_from_hexwkb", "tint_out")): + self.assertEqual(outs[writer], [(2, "size_t *")], writer) + h = self.eng.decode(reader, literal) + hexwkb = self.eng.encode(writer, h, [("int", 0)], outs[writer]) + self.assertRegex(hexwkb, r"^(?:[0-9A-F]{2})+$", writer) + aux = [("int", 15)] if out == "tint_out" else [] + again = self.eng.encode(out, self.eng.decode(back, hexwkb), aux) + self.assertEqual(again, self.eng.encode(out, h, aux), writer) + def test_bad_input_raises_not_exits(self): with self.assertRaises(MeosError): self.eng.decode("tbool_in", "not a temporal value at all") diff --git a/tests/test_server.py b/tests/test_server.py index 35cba11..efb109d 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -124,6 +124,7 @@ class FakeEngine(Engine): def __init__(self): self.calls = [] + self.outparams = [] self.fail = False def decode(self, fn, value, aux=()): @@ -137,8 +138,9 @@ def invoke(self, fn, args, rt): return {"void": None, "ptr": ("R", fn), "int": 42, "bool": 1, "double": 1.5, "str": "s"}[rt] - def encode(self, fn, handle, aux=()): + def encode(self, fn, handle, aux=(), outparams=()): self.calls.append(("encode", fn, handle, list(aux))) + self.outparams.append((fn, list(outparams))) return f"ENC({fn})" def invoke_outparam(self, fn, args, out_ctype, presence): @@ -326,6 +328,38 @@ def test_type_and_bool_rejection(self): {"temp": "x", "maxdd": True}) # bool != integer self.assertEqual(st, 400) + def test_the_writer_gets_the_out_parameters_its_catalog_entry_states(self): + """A result written by a function the catalog lists with an out-parameter + (``size_t *size_out`` of a hex-WKB writer) reaches the engine with its position + and type, as #test_routes_exclude_non_exposable reads the routes.""" + catalog = json.loads(json.dumps(CATALOG)) + catalog["functions"].append( + {"name": "temporal_out", "network": {"exposable": False}, + "params": [{"name": "temp", "cType": "const Temporal *"}, + {"name": "variant", "cType": "uint8_t"}, + {"name": "size_out", "cType": "size_t *"}], + "shape": {"outParams": ["size_out"]}, + "wire": {"params": [], "result": {"kind": "unsupported"}}}) + routes = build_routes(catalog) + self.assertEqual(routes["/temporal_round"]["result"]["encode_outparams"], + [(2, "size_t *")]) + self.assertEqual(routes["/temporal_set_interp"]["result"]["encode_outparams"], + [(2, "size_t *")]) + self.assertEqual(build_routes(CATALOG)["/temporal_round"]["result"] + ["encode_outparams"], []) + srv = make_server(catalog, self.engine, "127.0.0.1", 0) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + c = HTTPConnection("127.0.0.1", srv.server_address[1], timeout=5) + c.request("POST", "/temporal_round", json.dumps({"temp": "x", "maxdd": 3}), + {"Content-Type": "application/json"}) + self.assertEqual(c.getresponse().status, 200) + c.close() + finally: + srv.shutdown() + srv.server_close() + self.assertEqual(self.engine.outparams, [("temporal_out", [(2, "size_t *")])]) + def test_bad_enum_value(self): st, b = self.req("POST", "/temporal_set_interp", {"temp": "x", "interp": "NOPE"}) From 52df0f8d0060a5169a668c3a005c6abadec47afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Esteban=20Zim=C3=A1nyi?= Date: Sat, 31 Oct 2026 18:05:01 +0100 Subject: [PATCH 2/2] Derive the validated OpenAPI catalog through provision-meos.sh openapi-validate.yml regenerates the catalog with tools/provision-meos.sh over the MobilityDB checkout, which runs run.py with MDB_SRC_ROOT naming it, so the @ingroup groups that make a function public are attached and the exposable functions are projected. The workflow refuses a document with no path, and a change to provision-meos.sh runs it. Why: IO-B1 of IO-PLAN.md in MobilityDB #2861. run.py reads the groups from MDB_SRC_ROOT, else from ./_mobilitydb; the workflow set neither, so every function read internal and run 41207016338 published a valid document of 0 paths ("Done: 6073 functions (0 stateless-exposable)", "0 paths, 0 schemas"). Measured: over MobilityDB master 1ff2270f7f, run.py over the source headers without MDB_SRC_ROOT states all 6077 functions internal, 0 exposable, and generate_openapi.py projects 0 paths; through provision-meos.sh the catalog states 3614 exposable functions and the document, valid OpenAPI 3.1, 3614 paths and 35 schemas. The check refuses the document of 0 paths. --- .github/workflows/openapi-validate.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/openapi-validate.yml b/.github/workflows/openapi-validate.yml index 1462202..88ff070 100644 --- a/.github/workflows/openapi-validate.yml +++ b/.github/workflows/openapi-validate.yml @@ -15,6 +15,7 @@ on: - 'meta/**' - 'generate_openapi.py' - 'run.py' + - 'tools/provision-meos.sh' - 'requirements.txt' - '.github/workflows/openapi-validate.yml' push: @@ -50,8 +51,6 @@ jobs: run: | git clone --depth 1 https://github.com/MobilityDB/MobilityDB \ "$RUNNER_TEMP/mobilitydb" - echo "MOBILITYDB_HEADERS=$RUNNER_TEMP/mobilitydb/meos/include" \ - >> "$GITHUB_ENV" - name: Install Python dependencies run: | @@ -59,8 +58,11 @@ jobs: pip install -r requirements.txt pip install openapi-spec-validator + # tools/provision-meos.sh is the one recipe of the catalog: it parses the source + # headers with MDB_SRC_ROOT naming the checkout, from which run.py reads the + # @ingroup groups that make a function public, and so exposable. - name: Regenerate the catalog (parse + reconcile + enrich in one step) - run: python3 run.py "$MOBILITYDB_HEADERS" + run: tools/provision-meos.sh --mdb-src "$RUNNER_TEMP/mobilitydb" - name: Project to OpenAPI 3.1 run: python3 generate_openapi.py @@ -74,6 +76,8 @@ jobs: # OpenAPIV31SpecValidator(spec).validate() raises on violation, # returns None on success. Works with openapi-spec-validator 0.9.x. OpenAPIV31SpecValidator(spec).validate() + # A document with no path is valid OpenAPI and projects nothing. + assert spec.get('paths'), 'meos-openapi.json exposes no path' print(f\"::notice::meos-openapi.json conforms to OpenAPI 3.1 — \" f\"{len(spec.get('paths', {}))} paths, \" f\"{len(spec.get('components', {}).get('schemas', {}))} schemas.\")