From 4214f1cae2d42efe200dcb391a6ca5c08e676b72 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 15 Oct 2026 10:42:00 +0200 Subject: [PATCH 1/2] getCCDBTimeMachineTimestamp.py: minor refactoring --- GRID/utils/getCCDBTimeMachineTimestamp.py | 79 ++++++++++++----------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index 3a90db77a..807e6bfd6 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -96,40 +96,17 @@ def describe_object(o): f"comment=\"{o.get('comment', '')}\"") -def main(): - parser = argparse.ArgumentParser( - description="Estimate a CCDB TimeMachine not-after timestamp for a run+pass, " - "from AliEn catalog timestamps of that pass's own reco output.") - parser.add_argument("--year", default=os.environ.get("ALIEN_JDL_LPMANCHORYEAR"), - help="data-taking year, e.g. 2023 (default: $ALIEN_JDL_LPMANCHORYEAR)") - parser.add_argument("--period", default=os.environ.get("ALIEN_JDL_LPMANCHORPRODUCTION"), - help="period name, e.g. LHC23s (default: $ALIEN_JDL_LPMANCHORPRODUCTION)") - parser.add_argument("--run", default=os.environ.get("ALIEN_JDL_LPMANCHORRUN"), - help="run number (default: $ALIEN_JDL_LPMANCHORRUN)") - parser.add_argument("--pass", dest="passname", default=os.environ.get("ALIEN_JDL_LPMANCHORPASSNAME"), - help="pass name, e.g. apass4 (default: $ALIEN_JDL_LPMANCHORPASSNAME)") - parser.add_argument("--samples", type=int, default=9, - help="number of per-timeframe output files to sample (default: 9)") - parser.add_argument("--ccdb-check-path", default="TPC/Calib/CorrectionMapV2", - help="CCDB path to cross-check the estimate against (default: TPC/Calib/CorrectionMapV2)") - parser.add_argument("--no-ccdb-check", action="store_true", - help="skip the CCDB cross-check") - args = parser.parse_args() - - missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)] - if missing: - parser.error("missing required value(s): " + ", ".join(missing)) - - pass_dir = f"/alice/data/{args.year}/{args.period}/{args.run}/{args.passname}/" +def get_CCDBTimeMachineTS(year, period, run, passname, samples=9, ccdb_check_path="TPC/Calib/CorrectionMapV2", ccdb_check=True): + pass_dir = f"/alice/data/{year}/{period}/{run}/{passname}/" print(f"[getCCDBTimeMachineTimestamp] Looking for TF output under {pass_dir}", file=sys.stderr) - matches = find_tf_output_files(pass_dir, args.run) + matches = find_tf_output_files(pass_dir, run) if not matches: - sys.exit(f"[getCCDBTimeMachineTimestamp] No o2_ctf_run{int(args.run):08d}_orbit* output found " - f"under {pass_dir}. Check year/period/run/pass, or this run+pass may not use this " - f"catalog layout. Refusing to guess a timestamp.") + sys.exit(f"[getCCDBTimeMachineTimestamp] No o2_ctf_run{int(run):08d}_orbit* output found " + f"under {pass_dir}. Check year/period/run/pass, or this run+pass may not use this " + f"catalog layout. Refusing to guess a timestamp.") - sampled = sample_evenly(matches, args.samples) + sampled = sample_evenly(matches, samples) ctimes = [] for lfn in sampled: ctime = stat_ctime(lfn) @@ -139,7 +116,7 @@ def main(): if len(ctimes) < 2: sys.exit(f"[getCCDBTimeMachineTimestamp] Only got {len(ctimes)} usable timestamp(s) out of " - f"{len(sampled)} sampled files ({len(matches)} found in total); refusing to guess.") + f"{len(sampled)} sampled files ({len(matches)} found in total); refusing to guess.") not_after = int(statistics.median(ctimes)) spread_h = (max(ctimes) - min(ctimes)) / 3.6e6 @@ -148,24 +125,54 @@ def main(): print(f"[getCCDBTimeMachineTimestamp] Recommended not-after: {not_after} ({human(not_after)})", file=sys.stderr) - if not args.no_ccdb_check: - check = ccdb_cross_check(args.ccdb_check_path, not_after) + if ccdb_check: + check = ccdb_cross_check(ccdb_check_path, not_after) if check is None or "error" in check: reason = check["error"] if check else "unknown error" - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path} skipped: {reason}", + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path} skipped: {reason}", file=sys.stderr) elif check["as_of"] is None: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path}: no object " + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}: no object " f"existed yet at the recommended timestamp (earliest is {describe_object(check['newest'])})", file=sys.stderr) else: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path}: as-of object is " + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}: as-of object is " f"{describe_object(check['as_of'])}", file=sys.stderr) if check["outdated"]: print(f"[getCCDBTimeMachineTimestamp] note: CCDB's current object for this window is newer " f"({describe_object(check['newest'])}) - it postdates this pass's processing and was " f"not what this run actually saw.", file=sys.stderr) + return not_after + + +def main(): + parser = argparse.ArgumentParser( + description="Estimate a CCDB TimeMachine not-after timestamp for a run+pass, " + "from AliEn catalog timestamps of that pass's own reco output.") + parser.add_argument("--year", default=os.environ.get("ALIEN_JDL_LPMANCHORYEAR"), + help="data-taking year, e.g. 2023 (default: $ALIEN_JDL_LPMANCHORYEAR)") + parser.add_argument("--period", default=os.environ.get("ALIEN_JDL_LPMANCHORPRODUCTION"), + help="period name, e.g. LHC23s (default: $ALIEN_JDL_LPMANCHORPRODUCTION)") + parser.add_argument("--run", default=os.environ.get("ALIEN_JDL_LPMANCHORRUN"), + help="run number (default: $ALIEN_JDL_LPMANCHORRUN)") + parser.add_argument("--pass", dest="passname", default=os.environ.get("ALIEN_JDL_LPMANCHORPASSNAME"), + help="pass name, e.g. apass4 (default: $ALIEN_JDL_LPMANCHORPASSNAME)") + parser.add_argument("--samples", type=int, default=9, + help="number of per-timeframe output files to sample (default: 9)") + parser.add_argument("--ccdb-check-path", default="TPC/Calib/CorrectionMapV2", + help="CCDB path to cross-check the estimate against (default: TPC/Calib/CorrectionMapV2)") + parser.add_argument("--no-ccdb-check", action="store_true", + help="skip the CCDB cross-check") + args = parser.parse_args() + + missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)] + if missing: + parser.error("missing required value(s): " + ", ".join(missing)) + + not_after = get_CCDBTimeMachineTS(args.year, args.period, args.run, args.passname, args.samples, + args.ccdb_check_path, not args.no_ccdb_check) + print(not_after) From b820139b8315485df719904830aa66de713a6e89 Mon Sep 17 00:00:00 2001 From: Ernst Hellbar Date: Thu, 15 Oct 2026 10:45:52 +0200 Subject: [PATCH 2/2] getCCDBTimeMachineTimestamp.py: use the run start timestamp for the final cross check --- GRID/utils/getCCDBTimeMachineTimestamp.py | 39 ++++++++++++++++------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/GRID/utils/getCCDBTimeMachineTimestamp.py b/GRID/utils/getCCDBTimeMachineTimestamp.py index 807e6bfd6..2ce86696b 100755 --- a/GRID/utils/getCCDBTimeMachineTimestamp.py +++ b/GRID/utils/getCCDBTimeMachineTimestamp.py @@ -68,11 +68,7 @@ def human(ts_ms): return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") -def ccdb_cross_check(path, not_after, timeout=15): - """Report which object a correctly time-machine-pinned query at not_after - would actually resolve to, and flag if CCDB has since been updated for the - same validity window. Never raises; returns None on any failure.""" - url = f"{CCDB_HOST}/browse/{path}/{not_after}" +def get_ccdb_objects(url, timeout = 15): req = urllib.request.Request(url, headers={"Accept": "application/json"}) try: with urllib.request.urlopen(req, timeout=timeout) as resp: @@ -81,14 +77,33 @@ def ccdb_cross_check(path, not_after, timeout=15): return {"error": str(exc)} objects = data.get("objects", []) + return objects + + +def get_run_start_tf(run): + run_info_url = f"{CCDB_HOST}/browse/RCT/Info/RunInformation/{run}" + objects = get_ccdb_objects(run_info_url) + + return objects[0]["STF"] + + +def ccdb_cross_check(run, path, not_after, timeout=15): + """Report which object a correctly time-machine-pinned query at not_after + would actually resolve to, and flag if CCDB has since been updated for the + same validity window. Never raises; returns None on any failure.""" + timestamp = get_run_start_tf(run) + url = f"{CCDB_HOST}/browse/{path}/{timestamp}" + objects = get_ccdb_objects(url) if not objects: return {"error": "no CCDB object covers this instant"} as_of = [o for o in objects if o.get("Created", 0) <= not_after] newest = max(objects, key=lambda o: o.get("Created", 0)) - pinned = max(as_of, key=lambda o: o.get("Created", 0)) if as_of else None - outdated = pinned is not None and newest["Created"] > pinned["Created"] - return {"newest": newest, "as_of": pinned, "outdated": outdated} + result = {"timestamp": timestamp, "newest": newest, "as_of": None, "outdated": False} + if as_of: + result["as_of"] = max(as_of, key=lambda o: o.get("Created", 0)) + result["outdated"] = newest["Created"] > result["as_of"]["Created"] + return result def describe_object(o): @@ -126,17 +141,17 @@ def get_CCDBTimeMachineTS(year, period, run, passname, samples=9, ccdb_check_pat file=sys.stderr) if ccdb_check: - check = ccdb_cross_check(ccdb_check_path, not_after) + check = ccdb_cross_check(run, ccdb_check_path, not_after) if check is None or "error" in check: reason = check["error"] if check else "unknown error" - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path} skipped: {reason}", + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]} skipped: {reason}", file=sys.stderr) elif check["as_of"] is None: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}: no object " + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]}: no object " f"existed yet at the recommended timestamp (earliest is {describe_object(check['newest'])})", file=sys.stderr) else: - print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}: as-of object is " + print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]}: as-of object is " f"{describe_object(check['as_of'])}", file=sys.stderr) if check["outdated"]: print(f"[getCCDBTimeMachineTimestamp] note: CCDB's current object for this window is newer "