Call each writer with its out-parameters and derive the validated OpenAPI catalog through provision-meos.sh

.github/workflows/openapi-validate.yml

@@ -15,6 +15,7 @@ on:
       - 'meta/**'
       - 'generate_openapi.py'
       - 'run.py'
+      - 'tools/provision-meos.sh'
       - 'requirements.txt'
       - '.github/workflows/openapi-validate.yml'
   push:
@@ -50,17 +51,18 @@ jobs:
         run: |
           git clone --depth 1 https://github.com/MobilityDB/MobilityDB \
             "$RUNNER_TEMP/mobilitydb"
-          echo "MOBILITYDB_HEADERS=$RUNNER_TEMP/mobilitydb/meos/include" \
-            >> "$GITHUB_ENV"
 
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install openapi-spec-validator
 
+      # tools/provision-meos.sh is the one recipe of the catalog: it parses the source
+      # headers with MDB_SRC_ROOT naming the checkout, from which run.py reads the
+      # @ingroup groups that make a function public, and so exposable.
       - name: Regenerate the catalog (parse + reconcile + enrich in one step)
-        run: python3 run.py "$MOBILITYDB_HEADERS"
+        run: tools/provision-meos.sh --mdb-src "$RUNNER_TEMP/mobilitydb"
 
       - name: Project to OpenAPI 3.1
         run: python3 generate_openapi.py
@@ -74,6 +76,8 @@ jobs:
           # OpenAPIV31SpecValidator(spec).validate() raises on violation,
           # returns None on success. Works with openapi-spec-validator 0.9.x.
           OpenAPIV31SpecValidator(spec).validate()
+          # A document with no path is valid OpenAPI and projects nothing.
+          assert spec.get('paths'), 'meos-openapi.json exposes no path'
           print(f\"::notice::meos-openapi.json conforms to OpenAPI 3.1 — \"
                 f\"{len(spec.get('paths', {}))} paths, \"
                 f\"{len(spec.get('components', {}).get('schemas', {}))} schemas.\")

server/app.py

@@ -39,9 +39,31 @@ def build_enum_map(catalog: dict) -> dict:
     }
 
 
+def writer_outparams(catalog: dict) -> dict:
+    """{writer: [(position, cType)]}: the out-parameters each function states in its
+    ``shape.outParams``, by position, the value at 0 (``set_as_hexwkb``:
+    ``[(2, "size_t *")]``), which #CtypesEngine.encode of server/engine.py fills."""
+    out = {}
+    for fn in catalog.get("functions", []):
+        names = set((fn.get("shape") or {}).get("outParams") or ())
+        if names:
+            out[fn["name"]] = [(i, p["cType"]) for i, p in enumerate(fn["params"])
+                               if p["name"] in names]
+    return out
+
+
+def _with_outparams(value: dict | None, outs: dict) -> dict | None:
+    """``value`` (a wire result or element) with ``encode_outparams``, the
+    out-parameters of the writer it names."""
+    if not value or not value.get("encode"):
+        return value
+    return {**value, "encode_outparams": outs.get(value["encode"], [])}
+
+
 def build_routes(catalog: dict) -> dict:
     """`/{fn}` → route descriptor, for every exposable function."""
     routes = {}
+    outs = writer_outparams(catalog)
     for fn in catalog.get("functions", []):
         if not fn.get("network", {}).get("exposable"):
             continue
@@ -70,10 +92,11 @@ def build_routes(catalog: dict) -> dict:
             "result": {"kind": r["kind"], "json": r.get("json"),
                        "encode": r.get("encode"),
                        "encode_aux": r.get("encode_aux", []),
+                       "encode_outparams": outs.get(r.get("encode"), []),
                        "from_outparam": r.get("from_outparam"),
                        "out_ctype": r.get("out_ctype"),
                        "presence_return": r.get("presence_return", False),
-                       "element": r.get("element"),
+                       "element": _with_outparams(r.get("element"), outs),
                        "count_outparam": r.get("count_outparam"),
                        "count_param": r.get("count_param")},
         }
@@ -165,7 +188,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict):
             return None
         if res["kind"] == "serialized":         # opaque out-param -> encode
             return {"result": engine.encode(
-                res["encode"], val, _aux_args(res.get("encode_aux", [])))}
+                res["encode"], val, _aux_args(res.get("encode_aux", [])),
+                res.get("encode_outparams", []))}
         return {"result": bool(val) if res.get("json") == "boolean" else val}
 
     if res["kind"] == "array":
@@ -181,7 +205,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict):
             # Elem **f(.., int *count): MEOS returns a fresh array + byref count.
             ptrs = engine.invoke_array(route["name"], args)
         return {"result": [engine.encode(el["encode"], p,
-                                         _aux_args(el.get("encode_aux", [])))
+                                         _aux_args(el.get("encode_aux", [])),
+                                         el.get("encode_outparams", []))
                            for p in ptrs]}
 
     rtag = "void" if res["kind"] == "void" else (
@@ -193,7 +218,8 @@ def _dispatch(body: dict, route: dict, engine: Engine, enums: dict):
         return None
     if res["kind"] == "serialized":
         return {"result": engine.encode(
-            res["encode"], out, _aux_args(res.get("encode_aux", [])))}
+            res["encode"], out, _aux_args(res.get("encode_aux", [])),
+            res.get("encode_outparams", []))}
     if res["kind"] == "json" and res["json"] == "boolean":
         return {"result": bool(out)}
     return {"result": out}

server/engine.py

@@ -22,7 +22,7 @@
 
     decode(fn_name, value:str)            -> handle          (opaque)
     invoke(fn_name, args:[(tag,val)], rt) -> handle|scalar|None
-    encode(fn_name, handle)               -> str
+    encode(fn_name, handle, aux, outparams) -> str
 """
 
 from __future__ import annotations
@@ -46,7 +46,9 @@ def decode(self, fn_name: str, value: str, aux=()):
     def invoke(self, fn_name: str, args: list, result_tag: str):
         raise NotImplementedError
 
-    def encode(self, fn_name: str, handle, aux=()) -> str:
+    def encode(self, fn_name: str, handle, aux=(), outparams=()) -> str:
+        """``outparams``: the ``(position, cType)`` of the writer's out-parameters,
+        from the catalog's ``shape.outParams`` (#writer_outparams of server/app.py)."""
         raise NotImplementedError
 
     def invoke_outparam(self, fn_name: str, args: list, out_ctype: str,
@@ -83,7 +85,7 @@ def invoke(self, fn_name, args, result_tag):
         return {"str": "", "int": 0, "double": 0.0,
                 "bool": False, "enum": ""}.get(result_tag, "")
 
-    def encode(self, fn_name, handle, aux=()):
+    def encode(self, fn_name, handle, aux=(), outparams=()):
         return f"<stub:{fn_name}>"
 
     def invoke_outparam(self, fn_name, args, out_ctype, presence):
@@ -198,15 +200,47 @@ def invoke(self, fn_name, args, result_tag):
         self._raise_if_error()
         return r
 
-    def encode(self, fn_name, handle, aux=()):
+    def encode(self, fn_name, handle, aux=(), outparams=()):
+        """``char *f(const T *, aux..., size_t *size_out)``: the writer called with its
+        own signature. ``outparams`` are the ``(position, cType)`` of the parameters
+        the catalog states in its ``shape.outParams``, positions counted from the value
+        at 0; each gets a slot of its own, as #invoke_outparam gives its result one,
+        and the trailing inputs fill the other positions in order. A writer handing
+        back its size in a ``size_t *`` is read for that many bytes."""
         self._last_error = None
         f = self._sym(fn_name)
+        ct = self._ct
         atypes, avals = self._aux_ctypes(aux)
-        f.argtypes = [self._ct.c_void_p, *atypes]
-        f.restype = self._ct.c_char_p
-        r = f(handle, *avals)
+        outs = dict(outparams)
+        argtypes, cargs, slots = [ct.c_void_p], [handle], {}
+        rest = iter(zip(atypes, avals))
+        for pos in range(1, 1 + len(atypes) + len(outs)):
+            if pos in outs:
+                slot = self._pointee_ctype(outs[pos])()
+                slots[pos] = (outs[pos], slot)
+                argtypes.append(ct.POINTER(type(slot)))
+                cargs.append(ct.byref(slot))
+            else:
+                t, v = next(rest)
+                argtypes.append(t)
+                cargs.append(v)
+        f.argtypes = argtypes
+        if not slots:
+            f.restype = ct.c_char_p
+            r = f(*cargs)
+            self._raise_if_error()
+            return r.decode() if r else None
+        f.restype = ct.c_void_p
+        r = f(*cargs)
         self._raise_if_error()
-        return r.decode() if r else None
+        if not r:
+            return None
+        sizes = [s.value for c, s in slots.values()
+                 if " ".join(c.replace("*", " ").split()) == "size_t"]
+        raw = ct.string_at(r, sizes[0]) if sizes else ct.string_at(r)
+        # The size of a string MEOS writes counts its terminating NUL
+        # (``datum_as_wkb`` sizes a hex buffer ``2 * size + 1``).
+        return (raw[:-1] if raw.endswith(b"\0") else raw).decode()
 
     def _pointee_ctype(self, out_ctype: str):
         ct = self._ct
@@ -217,7 +251,7 @@ def _pointee_ctype(self, out_ctype: str):
             "long long": ct.c_longlong, "unsigned long": ct.c_ulong,
             "long": ct.c_long, "unsigned int": ct.c_uint,
             "short": ct.c_short, "unsigned char": ct.c_ubyte,
-            "signed char": ct.c_byte,
+            "signed char": ct.c_byte, "size_t": ct.c_size_t,
         }.get(base, ct.c_int)
 
     def invoke_outparam(self, fn_name, args, out_ctype, presence):

tests/test_engine_integration.py

@@ -179,6 +179,28 @@ def test_the_service_wire_keeps_the_srid(self):
                                 [("ptr", self.eng.decode(te["in"], plain))], "int")
         self.assertEqual(srid0, 0)
 
+    def test_each_hex_writer_round_trips_through_encode(self):
+        """A Set, a Span and a Temporal written by their hex-WKB writer through
+        #CtypesEngine.encode, with the out-parameters the catalog states
+        (#writer_outparams of server/app.py), read back to the value, as
+        #test_each_type_round_trips_through_its_own_writer round-trips the text."""
+        from server.app import writer_outparams
+        outs = writer_outparams(json.loads(_CATALOG.read_text()))
+        for reader, literal, writer, back, out in (
+                ("intset_in", "{1, 2, 3}", "set_as_hexwkb", "set_from_hexwkb",
+                 "intset_out"),
+                ("intspan_in", "[1, 5)", "span_as_hexwkb", "span_from_hexwkb",
+                 "intspan_out"),
+                ("tint_in", "{1@2000-01-01, 2@2000-01-02}", "temporal_as_hexwkb",
+                 "temporal_from_hexwkb", "tint_out")):
+            self.assertEqual(outs[writer], [(2, "size_t *")], writer)
+            h = self.eng.decode(reader, literal)
+            hexwkb = self.eng.encode(writer, h, [("int", 0)], outs[writer])
+            self.assertRegex(hexwkb, r"^(?:[0-9A-F]{2})+$", writer)
+            aux = [("int", 15)] if out == "tint_out" else []
+            again = self.eng.encode(out, self.eng.decode(back, hexwkb), aux)
+            self.assertEqual(again, self.eng.encode(out, h, aux), writer)
+
     def test_bad_input_raises_not_exits(self):
         with self.assertRaises(MeosError):
             self.eng.decode("tbool_in", "not a temporal value at all")

tests/test_server.py

@@ -124,6 +124,7 @@ class FakeEngine(Engine):
 
     def __init__(self):
         self.calls = []
+        self.outparams = []
         self.fail = False
 
     def decode(self, fn, value, aux=()):
@@ -137,8 +138,9 @@ def invoke(self, fn, args, rt):
         return {"void": None, "ptr": ("R", fn), "int": 42,
                 "bool": 1, "double": 1.5, "str": "s"}[rt]
 
-    def encode(self, fn, handle, aux=()):
+    def encode(self, fn, handle, aux=(), outparams=()):
         self.calls.append(("encode", fn, handle, list(aux)))
+        self.outparams.append((fn, list(outparams)))
         return f"ENC({fn})"
 
     def invoke_outparam(self, fn, args, out_ctype, presence):
@@ -326,6 +328,38 @@ def test_type_and_bool_rejection(self):
                          {"temp": "x", "maxdd": True})   # bool != integer
         self.assertEqual(st, 400)
 
+    def test_the_writer_gets_the_out_parameters_its_catalog_entry_states(self):
+        """A result written by a function the catalog lists with an out-parameter
+        (``size_t *size_out`` of a hex-WKB writer) reaches the engine with its position
+        and type, as #test_routes_exclude_non_exposable reads the routes."""
+        catalog = json.loads(json.dumps(CATALOG))
+        catalog["functions"].append(
+            {"name": "temporal_out", "network": {"exposable": False},
+             "params": [{"name": "temp", "cType": "const Temporal *"},
+                        {"name": "variant", "cType": "uint8_t"},
+                        {"name": "size_out", "cType": "size_t *"}],
+             "shape": {"outParams": ["size_out"]},
+             "wire": {"params": [], "result": {"kind": "unsupported"}}})
+        routes = build_routes(catalog)
+        self.assertEqual(routes["/temporal_round"]["result"]["encode_outparams"],
+                         [(2, "size_t *")])
+        self.assertEqual(routes["/temporal_set_interp"]["result"]["encode_outparams"],
+                         [(2, "size_t *")])
+        self.assertEqual(build_routes(CATALOG)["/temporal_round"]["result"]
+                         ["encode_outparams"], [])
+        srv = make_server(catalog, self.engine, "127.0.0.1", 0)
+        threading.Thread(target=srv.serve_forever, daemon=True).start()
+        try:
+            c = HTTPConnection("127.0.0.1", srv.server_address[1], timeout=5)
+            c.request("POST", "/temporal_round", json.dumps({"temp": "x", "maxdd": 3}),
+                      {"Content-Type": "application/json"})
+            self.assertEqual(c.getresponse().status, 200)
+            c.close()
+        finally:
+            srv.shutdown()
+            srv.server_close()
+        self.assertEqual(self.engine.outparams, [("temporal_out", [(2, "size_t *")])])
+
     def test_bad_enum_value(self):
         st, b = self.req("POST", "/temporal_set_interp",
                          {"temp": "x", "interp": "NOPE"})