Quickstart

1import os, urllib.request
2from margen import Margen
3 
4client = Margen(bearer_auth="mgn_test_...")   # free test key from the portal
5 
6# The passport test set is a free sample: one real and one fake per demographic
7# cell, from both editors (see Benchmarks). Pull one HiDream fake:
8item = client.list_items(
9    benchmark="passport-pad-v1", kind="fake", generator="hidream_o1", limit=1,
10).result.data[0]
11 
12dl = client.download_item(item_id=item.id)   # test tier: free, no credit debited
13print("signed url:", dl.url)                 # click to open the image in a browser
14 
15out_dir = "margen_out"                        # <- change to any folder you like
16os.makedirs(out_dir, exist_ok=True)
17path = os.path.join(out_dir, f"{item.id}.jpg")
18urllib.request.urlretrieve(dl.url, path)      # signed URL, no auth header
19print("saved to:", path)

1"""Pull a whole benchmark to a local folder, loadable exactly like a HuggingFace dataset."""
2import csv, os
3from margen import Margen
4from margen.ergonomics import iter_items, download_selection
5 
6BENCHMARK = "passport-pad-v1"
7OUT = "margen_passport"
8 
9client = Margen(bearer_auth=os.environ["MARGEN_API_KEY"])
10os.makedirs(OUT, exist_ok=True)
11 
12# 1. Catalogue every item with its full label object. Browsing labels is FREE:
13#    no bytes leave the server, no credits are debited. You get the whole table
14#    even if you never download an image.
15catalog = list(iter_items(client, benchmark=BENCHMARK, include="metadata"))
16print(f"catalogue: {len(catalog)} items")
17 
18# 1b. PREFLIGHT. Every image is one API call, so the key's rate limit sets the floor
19#     on wall-clock. A 429 is treated as skippable by the downloader, which means a
20#     limit that is too low does NOT fail loudly: the run finishes having fetched a
21#     fraction of the set. Check first and refuse to start.
22u = client.get_usage().result
23per_min = getattr(u, "rate_limit_per_min", 60) or 60
24hours = len(catalog) / per_min / 60
25if hours > 8:
26    raise SystemExit(
27        f"{len(catalog)} images at {per_min}/min would take {hours:.1f}h.
28"
29        f"Raise this key's rate limit in the portal (max 600/min, "
30        f"about {len(catalog) / 600 / 60:.1f}h) before starting."
31    )
32if u.balance < len(catalog):
33    print(f"[warn] balance {u.balance} < {len(catalog)} images; "
34          f"the pull will stop when credits run out (re-run after topping up)")
35print(f"rate limit {per_min}/min, estimated {hours:.1f}h")
36 
37# 2. Download only what this account does not already hold. Re-running is safe
38#    and cheap: owned images are skipped, so after topping up credits you just
39#    run it again and it fills the gap. Stops cleanly at a zero balance.
40todo = list(iter_items(client, benchmark=BENCHMARK, exclude_owned=True))
41print(f"to fetch: {len(todo)} (already owned: {len(catalog) - len(todo)})")
42download_selection(client, todo, OUT)
43 
44# 3. Write metadata.csv describing whatever is ACTUALLY on disk, including files
45#    from earlier runs. Matching on the id suffix rather than recomputing the
46#    filename means this stays correct if the naming convention ever changes.
47on_disk = {}
48for fn in os.listdir(OUT):
49    if fn.endswith(".jpg"):
50        on_disk[fn[:-4].rsplit("_", 1)[-1]] = fn
51 
52def val(v):
53    return getattr(v, "value", v)  # enum -> its value
54 
55def row_for(it, filename):
56    md = getattr(it, "metadata", None) or {}
57    if not isinstance(md, dict):
58        md = {k: v for k, v in vars(md).items() if not k.startswith("_")}
59    row = {
60        # file_name FIRST: this is what makes the folder a HuggingFace imagefolder.
61        "file_name": filename,
62        "label": 1 if val(getattr(it, "kind", None)) == "fake" else 0,
63        "id": str(it.id),
64        "kind": val(getattr(it, "kind", None)),
65        "skin_tone": val(getattr(it, "skin_tone", None)),
66        "gender": val(getattr(it, "gender", None)),
67        "generator": val(getattr(it, "generator", None)),
68        "perturbation": val(getattr(it, "perturbation", None)) or "clean",
69        # identity_id groups images of the SAME PERSON. Cluster your confidence
70        # intervals on this, not on row count, or they will be optimistic.
71        "identity_id": getattr(it, "identity_id", None),
72        "source_real_id": getattr(it, "source_real_id", None),
73    }
74    row.update({f"meta_{k}": v for k, v in md.items()})
75    return row
76 
77rows = [row_for(it, on_disk[str(it.id)[:8]]) for it in catalog if str(it.id)[:8] in on_disk]
78cols = list(dict.fromkeys(k for r in rows for k in r))
79with open(os.path.join(OUT, "metadata.csv"), "w", newline="") as fh:
80    w = csv.DictWriter(fh, fieldnames=cols, extrasaction="ignore")
81    w.writeheader()
82    w.writerows(rows)
83 
84print(f"{len(rows)} images + metadata.csv -> {OUT}/")
85print(f'load it:  datasets.load_dataset("imagefolder", data_dir="{OUT}")')