Skip to content

getCCDBTimeMachineTimestamp.py: use the run start timestamp for the final cross check #2507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ehellbar wants to merge 2 commits into from
Closed

getCCDBTimeMachineTimestamp.py: use the run start timestamp for the final cross check #2507

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4214f1c
getCCDBTimeMachineTimestamp.py: minor refactoring
ehellbar Oct 15, 2026
b820139
getCCDBTimeMachineTimestamp.py: use the run start timestamp for the f…
ehellbar Oct 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 66 additions & 44 deletions GRID/utils/getCCDBTimeMachineTimestamp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,7 @@
return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")


def ccdb_cross_check(path, not_after, timeout=15):
"""Report which object a correctly time-machine-pinned query at not_after
would actually resolve to, and flag if CCDB has since been updated for the
same validity window. Never raises; returns None on any failure."""
url = f"{CCDB_HOST}/browse/{path}/{not_after}"
def get_ccdb_objects(url, timeout = 15):
req = urllib.request.Request(url, headers={"Accept": "application/json"})
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
Expand All @@ -81,55 +77,51 @@
return {"error": str(exc)}

objects = data.get("objects", [])
return objects


def get_run_start_tf(run):
run_info_url = f"{CCDB_HOST}/browse/RCT/Info/RunInformation/{run}"
objects = get_ccdb_objects(run_info_url)

return objects[0]["STF"]


def ccdb_cross_check(run, path, not_after, timeout=15):
"""Report which object a correctly time-machine-pinned query at not_after
would actually resolve to, and flag if CCDB has since been updated for the
same validity window. Never raises; returns None on any failure."""
timestamp = get_run_start_tf(run)
url = f"{CCDB_HOST}/browse/{path}/{timestamp}"
objects = get_ccdb_objects(url)
if not objects:
return {"error": "no CCDB object covers this instant"}

as_of = [o for o in objects if o.get("Created", 0) <= not_after]
newest = max(objects, key=lambda o: o.get("Created", 0))
pinned = max(as_of, key=lambda o: o.get("Created", 0)) if as_of else None
outdated = pinned is not None and newest["Created"] > pinned["Created"]
return {"newest": newest, "as_of": pinned, "outdated": outdated}
result = {"timestamp": timestamp, "newest": newest, "as_of": None, "outdated": False}
if as_of:
result["as_of"] = max(as_of, key=lambda o: o.get("Created", 0))
result["outdated"] = newest["Created"] > result["as_of"]["Created"]

Check failure on line 105 in GRID/utils/getCCDBTimeMachineTimestamp.py

View workflow job for this annotation

GitHub Actions / Pylint

Pylint error unsubscriptable-object 

Value 'result['as_of']' is unsubscriptable
return result


def describe_object(o):
return (f"Created {human(o['Created'])}, JIRA={o.get('JIRA', '?')}, "
f"comment=\"{o.get('comment', '')}\"")


def main():
parser = argparse.ArgumentParser(
description="Estimate a CCDB TimeMachine not-after timestamp for a run+pass, "
"from AliEn catalog timestamps of that pass's own reco output.")
parser.add_argument("--year", default=os.environ.get("ALIEN_JDL_LPMANCHORYEAR"),
help="data-taking year, e.g. 2023 (default: $ALIEN_JDL_LPMANCHORYEAR)")
parser.add_argument("--period", default=os.environ.get("ALIEN_JDL_LPMANCHORPRODUCTION"),
help="period name, e.g. LHC23s (default: $ALIEN_JDL_LPMANCHORPRODUCTION)")
parser.add_argument("--run", default=os.environ.get("ALIEN_JDL_LPMANCHORRUN"),
help="run number (default: $ALIEN_JDL_LPMANCHORRUN)")
parser.add_argument("--pass", dest="passname", default=os.environ.get("ALIEN_JDL_LPMANCHORPASSNAME"),
help="pass name, e.g. apass4 (default: $ALIEN_JDL_LPMANCHORPASSNAME)")
parser.add_argument("--samples", type=int, default=9,
help="number of per-timeframe output files to sample (default: 9)")
parser.add_argument("--ccdb-check-path", default="TPC/Calib/CorrectionMapV2",
help="CCDB path to cross-check the estimate against (default: TPC/Calib/CorrectionMapV2)")
parser.add_argument("--no-ccdb-check", action="store_true",
help="skip the CCDB cross-check")
args = parser.parse_args()

missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)]
if missing:
parser.error("missing required value(s): " + ", ".join(missing))

pass_dir = f"/alice/data/{args.year}/{args.period}/{args.run}/{args.passname}/"
def get_CCDBTimeMachineTS(year, period, run, passname, samples=9, ccdb_check_path="TPC/Calib/CorrectionMapV2", ccdb_check=True):
pass_dir = f"/alice/data/{year}/{period}/{run}/{passname}/"
print(f"[getCCDBTimeMachineTimestamp] Looking for TF output under {pass_dir}", file=sys.stderr)

matches = find_tf_output_files(pass_dir, args.run)
matches = find_tf_output_files(pass_dir, run)
if not matches:
sys.exit(f"[getCCDBTimeMachineTimestamp] No o2_ctf_run{int(args.run):08d}_orbit* output found "
f"under {pass_dir}. Check year/period/run/pass, or this run+pass may not use this "
f"catalog layout. Refusing to guess a timestamp.")
sys.exit(f"[getCCDBTimeMachineTimestamp] No o2_ctf_run{int(run):08d}_orbit* output found "
f"under {pass_dir}. Check year/period/run/pass, or this run+pass may not use this "
f"catalog layout. Refusing to guess a timestamp.")

sampled = sample_evenly(matches, args.samples)
sampled = sample_evenly(matches, samples)
ctimes = []
for lfn in sampled:
ctime = stat_ctime(lfn)
Expand All @@ -139,7 +131,7 @@

if len(ctimes) < 2:
sys.exit(f"[getCCDBTimeMachineTimestamp] Only got {len(ctimes)} usable timestamp(s) out of "
f"{len(sampled)} sampled files ({len(matches)} found in total); refusing to guess.")
f"{len(sampled)} sampled files ({len(matches)} found in total); refusing to guess.")

not_after = int(statistics.median(ctimes))
spread_h = (max(ctimes) - min(ctimes)) / 3.6e6
Expand All @@ -148,24 +140,54 @@
print(f"[getCCDBTimeMachineTimestamp] Recommended not-after: {not_after} ({human(not_after)})",
file=sys.stderr)

if not args.no_ccdb_check:
check = ccdb_cross_check(args.ccdb_check_path, not_after)
if ccdb_check:
check = ccdb_cross_check(run, ccdb_check_path, not_after)
if check is None or "error" in check:
reason = check["error"] if check else "unknown error"
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path} skipped: {reason}",
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]} skipped: {reason}",
file=sys.stderr)
elif check["as_of"] is None:
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path}: no object "
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]}: no object "
f"existed yet at the recommended timestamp (earliest is {describe_object(check['newest'])})",
file=sys.stderr)
else:
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {args.ccdb_check_path}: as-of object is "
print(f"[getCCDBTimeMachineTimestamp] CCDB cross-check on {ccdb_check_path}/{check["timestamp"]}: as-of object is "
f"{describe_object(check['as_of'])}", file=sys.stderr)
if check["outdated"]:
print(f"[getCCDBTimeMachineTimestamp] note: CCDB's current object for this window is newer "
f"({describe_object(check['newest'])}) - it postdates this pass's processing and was "
f"not what this run actually saw.", file=sys.stderr)

return not_after


def main():
parser = argparse.ArgumentParser(
description="Estimate a CCDB TimeMachine not-after timestamp for a run+pass, "
"from AliEn catalog timestamps of that pass's own reco output.")
parser.add_argument("--year", default=os.environ.get("ALIEN_JDL_LPMANCHORYEAR"),
help="data-taking year, e.g. 2023 (default: $ALIEN_JDL_LPMANCHORYEAR)")
parser.add_argument("--period", default=os.environ.get("ALIEN_JDL_LPMANCHORPRODUCTION"),
help="period name, e.g. LHC23s (default: $ALIEN_JDL_LPMANCHORPRODUCTION)")
parser.add_argument("--run", default=os.environ.get("ALIEN_JDL_LPMANCHORRUN"),
help="run number (default: $ALIEN_JDL_LPMANCHORRUN)")
parser.add_argument("--pass", dest="passname", default=os.environ.get("ALIEN_JDL_LPMANCHORPASSNAME"),
help="pass name, e.g. apass4 (default: $ALIEN_JDL_LPMANCHORPASSNAME)")
parser.add_argument("--samples", type=int, default=9,
help="number of per-timeframe output files to sample (default: 9)")
parser.add_argument("--ccdb-check-path", default="TPC/Calib/CorrectionMapV2",
help="CCDB path to cross-check the estimate against (default: TPC/Calib/CorrectionMapV2)")
parser.add_argument("--no-ccdb-check", action="store_true",
help="skip the CCDB cross-check")
args = parser.parse_args()

missing = [n for n in ("year", "period", "run", "passname") if not getattr(args, n)]
if missing:
parser.error("missing required value(s): " + ", ".join(missing))

not_after = get_CCDBTimeMachineTS(args.year, args.period, args.run, args.passname, args.samples,
args.ccdb_check_path, not args.no_ccdb_check)

print(not_after)


Expand Down
Loading