#!/usr/bin/env python3
# GENERATED by factory-web/tools/build_agent.py from agent.py + extract.py: edit those, not this.
"""Gitorio agent: reads your local git repositories and keeps your hosted factory up to date.

    python gitorio.py --server https://gitorio.com --key gtr_...     (first run: saves both)
    python gitorio.py                     sync once
    python gitorio.py watch               sync every few minutes
    python gitorio.py sync --dry-run      show exactly what WOULD be sent, send nothing
    python gitorio.py add C:\\Dev\\project   also track a repository outside your scan folders
    python gitorio.py status              show what this agent is configured with

How it works: it pulls your settings from the website (which repos to track, hidden folders, ignore rules), finds
your repositories, builds a filtered description of each commit ON THIS MACHINE, and sends only the new ones. Your
code and git objects never leave your computer. Only read-only git commands are run.

Standard library only. Requires git on the PATH.
"""
import argparse, gzip, hashlib, json, os, re, sys, time, urllib.error, urllib.request
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

# --- bundled analysis module (extract.py), loaded under its usual name ---
import types as _types
extract = _types.ModuleType('extract')
extract.__file__ = __file__
sys.modules['extract'] = extract
exec(compile('#!/usr/bin/env python3\n"""Build factory-viz/timeline.json from one or more git repositories.\n\n    python extract.py                 build from config.json (created on first run)\n    python extract.py <repo> [...]    build from just these repos (single/monorepo auto-detected)\n\nThe settings modal in the viewer drives the same code through server.py.\n\nHow repos become sites:\n  mode "monorepo"  each top-level folder is a site (files in the root go to a "(root)" site)\n  mode "single"    the whole repo is one site, named after the repo\n  mode "auto"      guessed from the root: project manifests / few folders -> single, many plain folders -> monorepo\n\nOrder score (0..1) is computed over the merged stream, from cheap numstat-only signals:\n  test    tests / (tests + source)       focus   share of recent commits that stay inside one site\n  hotspot 1 - churn on files touched 4+ times     size   1 - share of files over 400 lines\n"""\nimport fnmatch, hashlib, json, os, re, subprocess, sys\nfrom concurrent.futures import ThreadPoolExecutor\nfrom collections import Counter, deque\nfrom datetime import datetime, timezone\nfrom pathlib import Path\n\nHERE = Path(__file__).parent\nCONFIG_PATH = HERE / "config.json"\nCACHE_DIR = HERE / "cache"\nOUT_PATH = HERE / "timeline.json"\n\nDEFAULT_IGNORE = [\n    "node_modules/", "vendor/", "dist/", "build/", ".git/", "__pycache__/", ".venv/", "venv/", ".idea/", ".vscode/",\n    "*.min.js", "*.min.css", "*.map", "package-lock.json", "composer.lock", "yarn.lock", "pnpm-lock.yaml", "poetry.lock",\n    "*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.ico", "*.bmp", "*.woff", "*.woff2", "*.ttf", "*.eot", "*.otf",\n    "*.zip", "*.gz", "*.tar", "*.7z", "*.pdf", "*.mp3", "*.mp4", "*.wav", "*.mov", "*.sqlite", "*.db", "*.exe", "*.dll",\n    "*.glb", "*.gltf", "*.fbx", "*.obj", "*.psd",            # 3D models / design binaries\n    "timeline.json", "cache/",                                # this tool\'s own generated output\n]\nDEFAULT_MAX_COMMITS = 5000\n\nSRC_EXT = {".py", ".php", ".js", ".ts", ".tsx", ".jsx", ".go", ".rs", ".java", ".cs", ".rb", ".c", ".cpp", ".h"}\nDOC_EXT = {".md", ".txt", ".rst"}\nCFG_EXT = {".json", ".yml", ".yaml", ".toml", ".ini", ".env", ".sql", ".xml"}\nWEB_EXT = {".html", ".css", ".scss"}\nTEST_RE = re.compile(r"(^|/)(tests?|__tests__|spec)(/|$)|(^|/)test_[^/]*$|[._-](test|spec)\\.[a-z]+$", re.I)\nWINDOW = 12   # commits considered "recent" for the order score\n\n# files whose presence at the repo root says "this is one project", not a folder of sites\nMANIFESTS = {"package.json", "composer.json", "pyproject.toml", "setup.py", "requirements.txt", "cargo.toml", "go.mod",\n             "gemfile", "manage.py", "app.py", "main.py", "index.php", "index.html", "pom.xml", "build.gradle", "makefile"}\nCONVENTIONAL_DIRS = {"src", "app", "lib", "tests", "test", "docs", "doc", "public", "static", "assets", "templates", "scripts",\n                     "config", "bin", "include", "cmd", "pkg", "internal", "migrations", "examples", "tools", "resources"}\n\n_NO_WINDOW = 0x08000000 if sys.platform == "win32" else 0\n\n\n# ---------------------------------------------------------------------------------------------\n# config\n# ---------------------------------------------------------------------------------------------\ndef slug(s):\n    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-") or "repo"\n\n\ndef repo_id(path):\n    key = os.path.normcase(os.path.realpath(path))\n    return f"{slug(Path(path).name)}-{hashlib.sha1(key.encode()).hexdigest()[:6]}"\n\n\ndef default_config():\n    cfg = {"version": 1, "maxCommits": DEFAULT_MAX_COMMITS, "ignoreDotFolders": True, "ignore": list(DEFAULT_IGNORE), "repos": []}\n    root = git_root(HERE)\n    if root:   # first run: start with the repo this tool lives in\n        cfg["repos"].append(make_repo_entry(root, mode="auto"))\n    return cfg\n\n\ndef clean_folder_list(items):\n    """Normalise a per-repo ignore list: strip slashes/whitespace/comments, drop blanks and duplicates."""\n    out, seen = [], set()\n    for x in items or []:\n        x = str(x).strip().replace("\\\\", "/").strip("/")\n        if not x or x.startswith("#") or len(x) > 200 or x.lower() in seen: continue\n        seen.add(x.lower()); out.append(x)\n    return out[:500]\n\n\ndef make_repo_entry(path, mode="auto", name=None, enabled=True, ignore_folders=None, ignore_root=False):\n    path = os.path.normpath(path)\n    return {"id": repo_id(path), "path": path, "name": name or Path(path).name, "mode": mode, "enabled": enabled,\n            "ignoreFolders": clean_folder_list(ignore_folders), "ignoreRoot": bool(ignore_root)}\n\n\ndef load_config(path=None):\n    path = Path(path or CONFIG_PATH)\n    if path.exists():\n        cfg = json.loads(path.read_text(encoding="utf-8"))\n    else:\n        cfg = default_config(); save_config(cfg, path)\n    cfg.setdefault("maxCommits", DEFAULT_MAX_COMMITS); cfg.setdefault("ignoreDotFolders", True)\n    cfg.setdefault("ignore", list(DEFAULT_IGNORE)); cfg.setdefault("repos", [])\n    return cfg\n\n\ndef save_config(cfg, path=None):\n    Path(path or CONFIG_PATH).write_text(json.dumps(cfg, indent=2), encoding="utf-8")\n\n\n# ---------------------------------------------------------------------------------------------\n# git helpers (no shell; arguments are passed as a list)\n# ---------------------------------------------------------------------------------------------\ndef git(root, *args, timeout=120):\n    r = subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, encoding="utf-8", errors="replace",\n                       timeout=timeout, creationflags=_NO_WINDOW)\n    if r.returncode != 0:\n        raise RuntimeError((r.stderr or r.stdout).strip() or f"git {args[0]} failed")\n    return r.stdout\n\n\ndef git_root(path):\n    try:\n        return os.path.normpath(git(path, "rev-parse", "--show-toplevel", timeout=10).strip())\n    except Exception:\n        return None\n\n\ndef suggest_mode(root):\n    """Guess monorepo vs single project from the repo root."""\n    try:\n        names, dirs = [], []\n        for line in git(root, "ls-tree", "HEAD", timeout=15).splitlines():   # "<mode> <type> <sha>\t<name>"\n            meta_, name = line.split("\\t", 1)\n            names.append(name)\n            if " tree " in meta_: dirs.append(name)\n    except Exception:\n        return "single", []\n    if any(n.lower() in MANIFESTS for n in names):\n        return "single", dirs\n    plain = [d for d in dirs if not d.startswith(".") and d.lower() not in CONVENTIONAL_DIRS]\n    return ("monorepo" if len(plain) >= 3 else "single"), dirs\n\n\ndef probe_repo(path):\n    """Validate a path and describe the repo it belongs to."""\n    path = os.path.normpath(os.path.expanduser(str(path).strip().strip(\'"\')))\n    if not os.path.isdir(path):\n        raise ValueError("That folder doesn\'t exist.")\n    root = git_root(path)\n    if not root:\n        raise ValueError("That folder isn\'t inside a git repository.")\n    try:\n        commits = int(git(root, "rev-list", "--count", "HEAD", timeout=30).strip())\n        branch = git(root, "rev-parse", "--abbrev-ref", "HEAD", timeout=10).strip()\n    except Exception:\n        raise ValueError("That repository has no commits yet.")\n    mode, dirs = suggest_mode(root)\n    return {"root": root, "name": Path(root).name, "commits": commits, "branch": branch, "suggestedMode": mode,\n            "topDirs": len(dirs), "insideRepo": os.path.normcase(os.path.realpath(path)) != os.path.normcase(os.path.realpath(root))}\n\n\nSKIP_DIRS = {"node_modules", "vendor", ".git", "venv", ".venv", "__pycache__", "dist", "build", ".cache", "appdata", "$recycle.bin"}\n\n\ndef scan_folder(path, max_depth=3, limit=200):\n    """Find git repositories under a folder (stops descending once it finds one)."""\n    path = os.path.normpath(os.path.expanduser(str(path).strip().strip(\'"\')))\n    if not os.path.isdir(path):\n        raise ValueError("That folder doesn\'t exist.")\n    found = []\n\n    def walk(d, depth):\n        if len(found) >= limit:\n            return\n        if os.path.exists(os.path.join(d, ".git")):\n            found.append(d); return\n        if depth >= max_depth:\n            return\n        try:\n            entries = sorted(os.scandir(d), key=lambda e: e.name.lower())\n        except OSError:\n            return\n        for e in entries:\n            if e.is_dir(follow_symlinks=False) and e.name.lower() not in SKIP_DIRS and not e.name.startswith("."):\n                walk(e.path, depth + 1)\n\n    walk(path, 0)\n\n    def light(d):   # two git calls per repo (commit count + root listing), run in parallel\n        try:\n            commits = int(git(d, "rev-list", "--count", "HEAD", timeout=30).strip())\n            return {"path": os.path.normpath(d), "name": Path(d).name, "commits": commits, "suggestedMode": suggest_mode(d)[0]}\n        except Exception:\n            return None   # empty or unreadable repos are skipped\n\n    with ThreadPoolExecutor(max_workers=8) as ex:\n        return [r for r in ex.map(light, found) if r]\n\n\n# ---------------------------------------------------------------------------------------------\n# raw extraction (per repo, cached by HEAD). Paths here are repo-relative; modes/ignores apply at merge time.\n# ---------------------------------------------------------------------------------------------\ndef _parse_rename(p):\n    m = re.match(r"^(.*)\\{(.*) => (.*)\\}(.*)$", p)   # a/{old => new}/b\n    if m:\n        pre, old, new, post = m.groups()\n        return (pre + old + post).replace("//", "/"), (pre + new + post).replace("//", "/")\n    if " => " in p:\n        old, new = p.split(" => ")\n        return old, new\n    return p, p\n\n\ndef extract_raw(root, max_commits):\n    out = git(root, "log", "--reverse", "--no-merges", "--numstat", "--summary", "-M", "-n", str(max_commits),\n              "--format=\\x1e%H\\x1f%aI\\x1f%an\\x1f%s", timeout=600)\n    commits = []\n    for block in out.split("\\x1e")[1:]:\n        head, *rest = block.split("\\n")\n        sha, date, author, subject = head.split("\\x1f", 3)\n        changes, deleted = [], set()\n        for line in rest:\n            if "\\t" in line:\n                a, d, p = line.split("\\t", 2)\n                old, new = _parse_rename(p)\n                changes.append([old, new, int(a) if a.isdigit() else 0, int(d) if d.isdigit() else 0])\n            elif line.startswith(" delete mode"):\n                deleted.add(line.split(" ", 4)[4])\n        for c in changes:\n            if c[1] in deleted:\n                c.append("D")\n        if changes:\n            commits.append({"sha": sha[:8], "date": date, "author": author, "subject": subject, "files": changes})\n    return commits\n\n\ndef get_raw(repo, max_commits, force=False):\n    """Return (commits, status) for one configured repo, using the cache when HEAD is unchanged."""\n    status = {"id": repo["id"], "name": repo["name"], "path": repo["path"]}\n    try:\n        root = git_root(repo["path"])\n        if not root:\n            raise RuntimeError("Not a git repository (or the folder is missing).")\n        head = git(root, "rev-parse", "HEAD", timeout=15).strip()\n        cache = CACHE_DIR / f"{repo[\'id\']}.json"\n        if not force and cache.exists():\n            c = json.loads(cache.read_text(encoding="utf-8"))\n            if c.get("head") == head and c.get("maxCommits") == max_commits:\n                status.update(head=head, commits=len(c["commits"]), syncedAt=c["syncedAt"], cached=True)\n                return c["commits"], status\n        commits = extract_raw(root, max_commits)\n        synced = datetime.now(timezone.utc).isoformat(timespec="seconds")\n        CACHE_DIR.mkdir(exist_ok=True)\n        cache.write_text(json.dumps({"head": head, "maxCommits": max_commits, "syncedAt": synced, "commits": commits}, separators=(",", ":")), encoding="utf-8")\n        status.update(head=head, commits=len(commits), syncedAt=synced, cached=False)\n        return commits, status\n    except Exception as e:\n        status.update(error=str(e)[:300])\n        return [], status\n\n\n# ---------------------------------------------------------------------------------------------\n# merge + order score\n# ---------------------------------------------------------------------------------------------\ndef kind(path):\n    if TEST_RE.search(path):\n        return "test"\n    ext = Path(path).suffix.lower()\n    if ext in SRC_EXT: return "src"\n    if ext in WEB_EXT: return "web"\n    if ext in DOC_EXT: return "doc"\n    if ext in CFG_EXT: return "cfg"\n    return "other"\n\n\ndef make_ignorer(patterns, dot_folders=True):\n    """Build ignored(path). dot_folders also skips anything inside a folder whose name starts with a dot\n    (.claude, .github, .vscode ...). Dot-FILES such as .gitignore or .env.example are still counted."""\n    dirs, globs = set(), []\n    for p in patterns:\n        p = p.strip().replace("\\\\", "/")\n        if not p or p.startswith("#"):\n            continue\n        if p.endswith("/"):\n            dirs.add(p.rstrip("/").lower())   # a folder name (or folder path) anywhere in the path\n        else:\n            globs.append(p.lower())           # a filename glob, or a path glob if it contains /\n    def ignored(path):\n        low = path.lower(); parts = low.split("/")\n        if dot_folders and any(part.startswith(".") for part in parts[:-1]):\n            return True\n        for d in dirs:\n            if "/" in d:\n                if low.startswith(d + "/") or ("/" + d + "/") in ("/" + low): return True\n            elif d in parts[:-1]:\n                return True\n        base = parts[-1]\n        return any(fnmatch.fnmatchcase(base, g) or ("/" in g and fnmatch.fnmatchcase(low, g)) for g in globs)\n    return ignored\n\n\ndef make_folder_ignorer(entries):\n    """Per-repo folder ignores. Each entry is a path relative to the repo root, matched segment by segment:\n    `scripts` hides only the top-level scripts folder (not src/scripts), `apps/legacy` hides that nested one, and\n    wildcards work within a segment (`*-old`, `tmp*`). A leading `**/` means "a folder like this anywhere" (`**/scratch`).\n    Everything inside a matched folder is skipped."""\n    rules = []   # (anywhere, [segment patterns])\n    for e in clean_folder_list(entries):\n        anywhere = e.startswith("**/")\n        rules.append((anywhere, (e[3:] if anywhere else e).lower().split("/")))\n    def ignored(path):\n        if not rules: return False\n        dirs = path.lower().split("/")[:-1]          # directories only: a file is hidden when a folder above it matches\n        for anywhere, segs in rules:\n            n = len(segs)\n            for start in (range(len(dirs) - n + 1) if anywhere else range(min(1, len(dirs) - n + 1))):\n                if all(fnmatch.fnmatchcase(dirs[start + k], segs[k]) for k in range(n)): return True\n        return False\n    return ignored\n\n\ndef list_folders(path, cfg=None):\n    """Top-level folders of a repo (at HEAD) with file counts, for the settings picker. Counts respect the global\n    ignore rules, but not the repo\'s own ignore list, so you can see (and un-hide) everything."""\n    root = git_root(path)\n    if not root:\n        raise ValueError("That folder isn\'t inside a git repository.")\n    cfg = cfg or {}\n    ign = make_ignorer(cfg.get("ignore", DEFAULT_IGNORE), cfg.get("ignoreDotFolders", True))\n    counts, root_files = Counter(), 0\n    for f in git(root, "ls-tree", "-r", "-z", "--name-only", "HEAD", timeout=120).split("\\0"):\n        if not f or ign(f): continue\n        parts = f.split("/")\n        if len(parts) > 1: counts[parts[0]] += 1\n        else: root_files += 1\n    return {"folders": [{"name": n, "files": c} for n, c in sorted(counts.items(), key=lambda kv: kv[0].lower())], "rootFiles": root_files}\n\n\ndef base_site(mode, repo_name, path):\n    """Which site a repo-relative path belongs to, and the path inside it: (site, rest)."""\n    if mode == "single":\n        return repo_name, path\n    parts = path.split("/")\n    return (parts[0], "/".join(parts[1:])) if len(parts) > 1 else ("(root)", path)\n\n\ndef plan_site_names(raws, skip):\n    """Site naming across repos. A site is (repo, folder); if two repos have a folder with the same name the sites are\n    disambiguated as "name (repo)". raws is a list of (order, repo, mode, commits). Returns final_site(base, repo_name)."""\n    seen_in = {}\n    for order, r, mode, commits in raws:\n        names = set()\n        for c in commits:\n            for f in c["files"]:\n                if not skip(r, f[1]):\n                    names.add(base_site(mode, r["name"], f[1])[0])\n        for n in names:\n            seen_in.setdefault(n, set()).add(r["id"])\n\n    def final_site(base, repo_name):\n        return f"{base} ({repo_name})" if len(seen_in.get(base, ())) > 1 else base\n    return final_site\n\n\ndef build_repo_steps(r, mode, commits, skip, final_site):\n    """Stage 1, per repository (this is what the agent pushes): replay one repo\'s commits into steps\n    {sha, date, author, subject, repo, add, del, ch}, with final site-prefixed paths. State is per repo, so\n    nothing here depends on any other repo. Returns (steps, site_info)."""\n    files, steps, site_info = {}, [], {}\n\n    def fp(path):\n        base, rest = base_site(mode, r["name"], path)\n        site = final_site(base, r["name"])\n        site_info.setdefault(site, {"repo": r["name"], "mode": mode})\n        return f"{site}/{rest}"\n\n    for c in commits:\n        added, removed, churn = [], [], []\n        for f in c["files"]:\n            old, new, a, d = f[0], f[1], f[2], f[3]\n            gone = len(f) > 4 and f[4] == "D"\n            if skip(r, old) and skip(r, new):\n                continue\n            # an ignored path never becomes a file or a site; moving a file into an ignored folder counts as removing it\n            pold = None if skip(r, old) else fp(old)\n            pnew = None if skip(r, new) else fp(new)\n            if gone or pnew is None:\n                if pold is not None and pold in files:\n                    del files[pold]\n                    if pold in added: added.remove(pold)\n                    else: removed.append(pold)\n                continue\n            if old != new and pold in files:\n                files[pnew] = files.pop(pold); removed.append(pold); added.append(pnew)\n            if pnew not in files:\n                files[pnew] = 0; added.append(pnew)\n            files[pnew] = max(0, files[pnew] + a - d)\n            churn.append(pnew)\n        if not (added or removed or churn):\n            continue\n        steps.append({\n            "sha": c["sha"], "date": c["date"], "author": c["author"], "subject": c["subject"], "repo": r["name"],\n            "add": added, "del": removed, "ch": [[p, files[p]] for p in dict.fromkeys(churn) if p in files],\n        })\n    return steps, site_info\n\n\ndef merge_and_score(per_repo):\n    """Stage 2 (the viewer does this in JS for hosted accounts): merge every repo\'s steps into one time-ordered\n    stream, replay the file state, and add the order score. per_repo is a list of steps lists in repo order.\n    Returns (steps, files)."""\n    stream = []\n    for order, steps in enumerate(per_repo):\n        for idx, s in enumerate(steps):\n            stream.append((datetime.fromisoformat(s["date"]).timestamp(), order, idx, s))\n    stream.sort(key=lambda t: t[:3])\n\n    files, touch_count, recent, out = {}, Counter(), deque(maxlen=WINDOW), []\n    for _, _, _, s in stream:\n        for p in s["del"]:\n            files.pop(p, None)\n        for p in s["add"]:\n            files.setdefault(p, 0)\n        churn = []\n        for p, n in s["ch"]:\n            files[p] = n; touch_count[p] += 1; churn.append(p)\n\n        sites = {p.split("/", 1)[0] for p in churn}\n        recent.append((1.0 if len(sites) <= 1 else 1.0 / len(sites), churn))\n\n        kinds = Counter(kind(p) for p in files)\n        tests, src = kinds["test"], kinds["src"] + kinds["web"]\n        test_score = min(1.0, (tests / (tests + src)) * 3) if (tests + src) else 0.0\n        focus_score = sum(x for x, _ in recent) / len(recent)\n        rp = [p for _, ps in recent for p in ps]\n        hotspot_score = 1 - (sum(1 for p in rp if touch_count[p] >= 4) / len(rp)) if rp else 1.0\n        size_score = 1 - (sum(1 for n in files.values() if n > 400) / len(files)) if files else 1.0\n        order_score = 0.25 * test_score + 0.30 * focus_score + 0.20 * hotspot_score + 0.25 * size_score\n        out.append({**s, "files": len(files), "order": round(order_score, 3),\n                    "parts": {"test": round(test_score, 2), "focus": round(focus_score, 2), "hotspot": round(hotspot_score, 2), "size": round(size_score, 2)}})\n    return out, files\n\n\ndef make_skipper(ignore_patterns, dot_folders, repos):\n    """skip(repo, path): True when a repo-relative path must be left out. Combines the global ignore rules, the\n    repo\'s own hidden folders, and (optionally) files sitting directly in the repo root. One definition, shared by the\n    local tool, the agent and the tests, so they can never disagree about what is ignored."""\n    ignored = make_ignorer(ignore_patterns, dot_folders)\n    folder_ign = {r["id"]: make_folder_ignorer(r.get("ignoreFolders", [])) for r in repos}\n\n    def skip(r, path):\n        return ignored(path) or folder_ign[r["id"]](path) or (bool(r.get("ignoreRoot")) and "/" not in path)\n    return skip\n\n\ndef build_timeline(cfg, force=False, out_path=OUT_PATH):\n    max_commits = int(cfg.get("maxCommits") or DEFAULT_MAX_COMMITS)\n    repos = [r for r in cfg["repos"] if r.get("enabled", True)]\n    skip = make_skipper(cfg.get("ignore", []), cfg.get("ignoreDotFolders", True), repos)\n\n    statuses, raws = {}, []\n    for r in cfg["repos"]:\n        if not r.get("enabled", True):\n            statuses[r["id"]] = {"id": r["id"], "name": r["name"], "path": r["path"], "disabled": True}\n    for order, r in enumerate(repos):\n        commits, st = get_raw(r, max_commits, force)\n        mode = r.get("mode", "auto")\n        if mode == "auto" and not st.get("error"):\n            mode = suggest_mode(git_root(r["path"]))[0]\n        st["mode"] = mode; statuses[r["id"]] = st\n        raws.append((order, r, mode, commits))\n\n    final_site = plan_site_names(raws, skip)\n    per_repo, site_info = [], {}\n    for order, r, mode, commits in raws:\n        steps, info = build_repo_steps(r, mode, commits, skip, final_site)\n        per_repo.append(steps)\n        for site, v in info.items():\n            site_info.setdefault(site, v)\n\n    steps, files = merge_and_score(per_repo)\n\n    meta = {p: {"k": kind(p), "n": n, "d": p.split("/", 1)[0]} for p, n in files.items()}\n    for s in steps:\n        for p in s["add"] + s["del"]:\n            meta.setdefault(p, {"k": kind(p), "n": 0, "d": p.split("/", 1)[0]})\n    label = repos[0]["name"] if len(repos) == 1 else (f"{len(repos)} repositories" if repos else "No repositories")\n    out = {"repo": label, "repos": [statuses[r["id"]] for r in cfg["repos"]], "sites": site_info, "steps": steps, "meta": meta}\n    Path(out_path).write_text(json.dumps(out, separators=(",", ":")), encoding="utf-8")\n    return {"steps": len(steps), "filesNow": len(files), "filesEver": len(meta), "sites": len({m["d"] for p, m in meta.items() if p in files}),\n            "repos": out["repos"]}\n\n\nif __name__ == "__main__":\n    args = [a for a in sys.argv[1:] if not a.startswith("--")]\n    if args:   # ad-hoc: build from just these paths without touching config.json\n        cfg = {"maxCommits": DEFAULT_MAX_COMMITS, "ignoreDotFolders": True, "ignore": list(DEFAULT_IGNORE), "repos": []}\n        for a in args:\n            root = git_root(a)\n            if not root: sys.exit(f"{a} is not inside a git repository")\n            cfg["repos"].append(make_repo_entry(root))\n    else:\n        cfg = load_config()\n    res = build_timeline(cfg, force="--force" in sys.argv)\n    for r in res["repos"]:\n        print(("  ! " if r.get("error") else "  - ") + r["name"] + ": " + (r.get("error") or f"{r.get(\'commits\', 0)} commits" + (" (cached)" if r.get("cached") else "")))\n    print(f"{res[\'steps\']} commits, {res[\'filesEver\']} files ever, {res[\'filesNow\']} now, {res[\'sites\']} sites -> {OUT_PATH}")\n', 'extract.py', 'exec'), extract.__dict__)
# --- end bundled module ---

AGENT_VERSION = "0.1.0"
CHUNK = 500                      # commits per request (the server accepts up to 1000)
HOME = Path(os.environ.get("GITORIO_AGENT_HOME") or Path.home() / ".gitorio")
STEP_FIELDS = ("sha", "date", "author", "subject", "repo", "add", "del", "ch")


def say(msg=""):
    print(msg, flush=True)


def vtuple(v):
    return tuple(int(x) for x in re.findall(r"\d+", str(v))[:3])


def load_json(path, default):
    try:
        return json.loads(Path(path).read_text(encoding="utf-8"))
    except (OSError, ValueError):
        return default


def save_json(path, obj):
    Path(path).parent.mkdir(parents=True, exist_ok=True)
    Path(path).write_text(json.dumps(obj, indent=2), encoding="utf-8")


class ApiError(Exception):
    def __init__(self, status, code, message):
        super().__init__(f"{status} {code}: {message}")
        self.status, self.code, self.message = status, code, message


class Api:
    def __init__(self, server, key):
        self.server, self.key = server.rstrip("/"), key

    def call(self, method, path, body=None):
        headers = {"Authorization": "Bearer " + self.key, "X-Agent-Version": AGENT_VERSION, "Accept": "application/json"}
        data = None
        if body is not None:
            raw = json.dumps(body, separators=(",", ":")).encode()
            data = gzip.compress(raw) if len(raw) > 1024 else raw
            headers["Content-Type"] = "application/json"
            if len(raw) > 1024:
                headers["Content-Encoding"] = "gzip"
        last = None
        for attempt in range(3):   # transient network or server trouble is retried; real answers are not
            try:
                with urllib.request.urlopen(urllib.request.Request(self.server + path, data=data, headers=headers, method=method), timeout=60) as r:
                    return json.loads(r.read() or b"{}")
            except urllib.error.HTTPError as e:
                try:
                    j = json.loads(e.read() or b"{}")
                except ValueError:
                    j = {}
                if e.code >= 500 and attempt < 2:
                    last = e; time.sleep(2 * (attempt + 1)); continue
                raise ApiError(e.code, j.get("error", "http_error"), j.get("message", e.reason))
            except (urllib.error.URLError, TimeoutError, ConnectionError) as e:
                last = e
                if attempt < 2:
                    time.sleep(2 * (attempt + 1)); continue
        raise ApiError(0, "network", f"could not reach {self.server}: {getattr(last, 'reason', last)}")


# ---------------------------------------------------------------------------------------------
# finding repositories
# ---------------------------------------------------------------------------------------------
def agent_repo_id(path):
    """Opaque, stable id for a repository, derived from its path. The server never learns the path itself."""
    return hashlib.sha1(os.path.normcase(os.path.realpath(path)).encode("utf-8")).hexdigest()[:12]


def discover(scan_roots, extra_paths, say=say):
    """{agent_repo_id: {path, name, commits, suggested_mode}} for every repo under the scan roots plus any added by hand."""
    found = {}
    for root in scan_roots:
        try:
            for r in extract.scan_folder(root):
                found[agent_repo_id(r["path"])] = {"path": r["path"], "name": r["name"], "commits": r["commits"], "suggested_mode": r["suggestedMode"]}
        except ValueError as e:
            say(f"  ! scan folder {root}: {e}")
    for p in extra_paths:
        try:
            info = extract.probe_repo(p)
            found[agent_repo_id(info["root"])] = {"path": info["root"], "name": info["name"], "commits": info["commits"], "suggested_mode": info["suggestedMode"]}
        except ValueError as e:
            say(f"  ! added repository {p}: {e}")
    return found


def display_path(path):
    parts = [x for x in Path(path).parts if x not in ("\\", "/")]
    return "/".join(parts[-2:])


def report_repos(api, found, state, cfg_like, defaults, force=False, say=say):
    """Tells the server which repositories exist (and their top-level folders) so the website can configure them."""
    def head_of(path):
        try:
            return extract.git(path, "rev-parse", "HEAD", timeout=15).strip()
        except Exception:
            return ""

    todo = {}
    with ThreadPoolExecutor(max_workers=8) as ex:
        heads = dict(zip(found, ex.map(lambda i: head_of(found[i]["path"]), found)))
    for rid, info in found.items():
        fingerprint = heads[rid] + "|" + json.dumps(cfg_like, sort_keys=True)
        if force or state.setdefault("repos", {}).get(rid, {}).get("reported") != fingerprint:
            todo[rid] = (info, fingerprint)

    payload = []
    def inventory(item):
        rid, (info, _) = item
        inv = extract.list_folders(info["path"], cfg_like)
        return {"id": rid, "name": info["name"], "display_path": display_path(info["path"]), "suggested_mode": info["suggested_mode"],
                "commits": info["commits"], "folders": inv["folders"], "root_files": inv["rootFiles"]}
    with ThreadPoolExecutor(max_workers=6) as ex:
        for item, entry in zip(todo.items(), ex.map(inventory, todo.items())):
            payload.append(entry)
    if not payload and not force:
        return 0
    res = api.call("POST", "/agent/v1/repos", {"repos": payload, "agent_defaults": defaults})
    for rid, (info, fingerprint) in todo.items():
        state["repos"].setdefault(rid, {})["reported"] = fingerprint
    say(f"  reported {len(payload)} repositor{'y' if len(payload) == 1 else 'ies'} ({res.get('added', 0)} new)")
    if res.get("skipped"):
        say(f"  ! the website skipped {res['skipped']} repositor{'y' if res['skipped'] == 1 else 'ies'} (account limit reached, or an invalid entry)")
        # forget the fingerprints of anything we reported so a later run (after the limit changes) tries again
        for rid in todo:
            state["repos"][rid].pop("reported", None)
    return len(payload)


# ---------------------------------------------------------------------------------------------
# building and pushing steps
# ---------------------------------------------------------------------------------------------
def anonymise(steps, salt):
    """Replaces file names with stable hashes (kind and size are kept, so the factory still renders) and drops commit
    messages and authors. The site name (the folder) is left as is; the website lets you alias it."""
    def conv(path):
        site, _, rest = path.partition("/")
        h = hashlib.sha1((salt + "/" + rest).encode()).hexdigest()[:10]
        # The extension and "is this a test?" are what decide a file's kind, so both survive; the name itself does not.
        marker = ".test" if extract.kind(path) == "test" else ""
        ext = os.path.splitext(rest)[1] or (".x" if marker else "")
        return f"{site}/{h}{marker}{ext}"
    out = []
    for s in steps:
        t = {k: s[k] for k in ("sha", "date", "repo") if k in s}
        t["add"] = [conv(p) for p in s["add"]]
        t["del"] = [conv(p) for p in s["del"]]
        t["ch"] = [[conv(p), n] for p, n in s["ch"]]
        out.append(t)
    return out


def build_plans(settings, server_repos, local, say=say):
    """For every repository enabled on the website and present on this machine, the steps to publish and the
    signature of the settings they were built with."""
    ignore = settings["ignore"] if settings.get("ignore") is not None else list(extract.DEFAULT_IGNORE)
    dot = bool(settings.get("ignore_dot_folders", True))
    max_commits = int(settings.get("max_commits") or extract.DEFAULT_MAX_COMMITS)
    extract.CACHE_DIR = HOME / "cache"

    enabled = [sr for sr in server_repos if sr["enabled"] and sr["id"] in local]
    missing = [sr["name"] for sr in server_repos if sr["enabled"] and sr["id"] not in local]
    if missing:
        say(f"  note: enabled on the website but not found on this computer: {', '.join(missing)}")
    repos = [{"id": sr["id"], "name": sr["name"], "path": local[sr["id"]]["path"], "ignoreFolders": sr["ignore_folders"],
              "ignoreRoot": sr["ignore_root"], "enabled": True} for sr in enabled]
    skip = extract.make_skipper(ignore, dot, repos)

    raws = []
    for order, (r, sr) in enumerate(zip(repos, enabled)):
        commits, st = extract.get_raw(r, max_commits)
        if st.get("error"):
            say(f"  ! {r['name']}: {st['error']}")
            continue
        mode = sr["mode"] if sr["mode"] != "auto" else extract.suggest_mode(extract.git_root(r["path"]))[0]
        raws.append((order, r, mode, commits, sr))
    final_site = extract.plan_site_names([(o, r, m, c) for o, r, m, c, _ in raws], skip)

    plans = []
    for order, r, mode, commits, sr in raws:
        steps, _ = extract.build_repo_steps(r, mode, commits, skip, final_site)
        bases = sorted({extract.base_site(mode, r["name"], f[1])[0] for c in commits for f in c["files"] if not skip(r, f[1])})
        naming = {b: final_site(b, r["name"]) for b in bases}
        sig_obj = {"v": 1, "mode": mode, "ignore": ignore, "dot": dot, "folders": sr["ignore_folders"], "root": sr["ignore_root"],
                   "detail": sr["detail"], "max": max_commits, "naming": naming}
        sig = hashlib.sha1(json.dumps(sig_obj, sort_keys=True).encode()).hexdigest()[:32]
        if sr["detail"] == "anonymous":
            steps = anonymise(steps, r["id"])
        else:
            steps = [{k: s[k] for k in STEP_FIELDS if k in s} for s in steps]
        plans.append({"repo": r, "server": sr, "steps": steps, "sig": sig, "mode": mode})
    return plans


def fit_chunk(repo_id, sig, steps, base, size, max_bytes):
    """The request body for steps[base:base+size], shrunk (by halving the commit count) until its COMPRESSED size fits
    under max_bytes. The limit matters because the web server rejects bodies over ~1 MB. Returns (body, size)."""
    while True:
        body = {"repo_id": repo_id, "sig": sig, "base_index": base, "prev_sha": steps[base - 1]["sha"] if base > 0 else "",
                "commits": steps[base:base + size]}
        wire = len(gzip.compress(json.dumps(body, separators=(",", ":")).encode()))
        if wire <= max_bytes or size <= 1:
            return body, size
        size = max(1, size // 2)


def push_plan(api, plan, say=say, limits=None):
    """Sends whatever the server doesn't have yet for one repository. Resumable: the server says where it is."""
    limits = limits or {}
    r, sr, steps, sig = plan["repo"], plan["server"], plan["steps"], plan["sig"]
    size = int(os.environ.get("GITORIO_AGENT_CHUNK") or limits.get("chunk_commits") or CHUNK)
    max_bytes = int(os.environ.get("GITORIO_AGENT_MAX_BODY") or limits.get("max_body_bytes") or 700000)
    stored = sr["stored"]
    base = stored["count"] if stored["sig"] == sig and stored["count"] <= len(steps) else 0
    same_tip = bool(steps) and stored["last_sha"] == steps[-1]["sha"]
    if base == len(steps) and ((stored["sig"] == sig and (same_tip or not steps)) or (not steps and stored["count"] == 0)):
        say(f"  {r['name']}: up to date ({len(steps)} commits)")
        return 0
    # (same count but a different last commit means history was rewritten: fall through, and the server will say so)

    sent, resets, guard = 0, 0, 0
    force_reset = stored["count"] > len(steps)   # the server holds more than we now have: history shrank
    while guard < len(steps) // max(1, min(size, CHUNK)) + 20:
        guard += 1
        body, size = fit_chunk(r["id"], sig, steps, base, size, max_bytes)   # `size` only ever shrinks within one push
        chunk = body["commits"]
        if force_reset:
            body["reset"] = True; force_reset = False
        res = api.call("POST", "/agent/v1/sync", body)
        status = res.get("status")
        if status == "ok":
            sent += len(chunk); base = res["next_index"]
            say(f"  {r['name']}: sent {base}/{len(steps)}")
            if base >= len(steps):
                return sent
        elif status in ("resync", "reset"):
            if status == "reset":
                resets += 1
                say(f"  {r['name']}: starting over ({res.get('reason', 'changed')})")
                if resets > 2:
                    raise ApiError(0, "reset_loop", "the server keeps resetting this repository")
            base = res["next_index"]
            if base > len(steps):
                force_reset = True; base = 0
            if not steps and status == "reset":
                return 0
        else:
            raise ApiError(0, "unexpected", f"unexpected answer: {res}")
    raise ApiError(0, "stuck", "could not finish syncing this repository")


# ---------------------------------------------------------------------------------------------
# commands
# ---------------------------------------------------------------------------------------------
def run_sync(cfg, dry_run=False, force_report=False):
    api = Api(cfg["server"], cfg["key"])
    hello = api.call("GET", "/agent/v1/hello")
    if vtuple(AGENT_VERSION) < vtuple(hello["min_agent_version"]):
        say(f"This agent (v{AGENT_VERSION}) is too old for the server (needs v{hello['min_agent_version']}+). Please download the latest agent.")
        return 2
    say(f"Connected as {hello['user']} (key: {hello['key_name']}){' [dry run: nothing will be sent]' if dry_run else ''}")
    if cfg.pop("_unsaved", False):   # credentials given on the command line are saved only once the server has accepted them
        save_json(HOME / "config.json", cfg)
        try: os.chmod(HOME / "config.json", 0o600)
        except OSError: pass

    settings_res = api.call("GET", "/agent/v1/settings")
    settings = settings_res["settings"]
    cfg_like = {"ignore": settings["ignore"] if settings.get("ignore") is not None else list(extract.DEFAULT_IGNORE),
                "ignoreDotFolders": bool(settings.get("ignore_dot_folders", True))}
    state = load_json(HOME / "state.json", {})

    found = discover(settings.get("scan_roots", []), cfg.get("extra_repos", []))
    say(f"Found {len(found)} repositor{'y' if len(found) == 1 else 'ies'} on this computer" + (f" in {len(settings.get('scan_roots', []))} scan folder(s)" if settings.get("scan_roots") else ""))
    if not found and not settings.get("scan_roots"):
        say("  No scan folders are set yet. Add one on the website (Settings), or run: agent.py add <path-to-a-repo>")

    if not dry_run:
        report_repos(api, found, state, cfg_like, {"ignore": list(extract.DEFAULT_IGNORE), "max_commits": extract.DEFAULT_MAX_COMMITS}, force=force_report)
        settings_res = api.call("GET", "/agent/v1/settings")   # newly reported repos now exist (disabled) on the server
    local = {rid: {"path": i["path"], "name": i["name"]} for rid, i in found.items()}
    state["local"] = {rid: v["path"] for rid, v in local.items()}

    plans = build_plans(settings_res["settings"], settings_res["repos"], local)
    enabled_names = [p["repo"]["name"] for p in plans]
    if not plans:
        say("No repositories are enabled yet. Turn some on at the website, then run the agent again.")
    total_sent = 0
    for plan in plans:
        r, steps, sr = plan["repo"], plan["steps"], plan["server"]
        if dry_run:
            stored = sr["stored"]; base = stored["count"] if stored["sig"] == plan["sig"] and stored["count"] <= len(steps) else 0
            say(f"  {r['name']}: {len(steps)} commits built, would send {len(steps) - base} (detail: {sr['detail']}, mode: {plan['mode']})")
            if steps[base:]:
                sample = steps[base]
                say("    first commit that would be sent: " + json.dumps(sample)[:300])
            continue
        total_sent += push_plan(api, plan, limits=settings_res.get("limits"))

    save_json(HOME / "state.json", state)
    if not dry_run:
        say(f"Done. {total_sent} commit(s) sent for {len(plans)} repositor{'y' if len(plans) == 1 else 'ies'}.")
    return 0


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("command", nargs="?", default="sync", choices=["sync", "watch", "add", "status"])
    ap.add_argument("path", nargs="?", help="for 'add': the repository folder")
    ap.add_argument("--server", help="website address, e.g. https://example.com (saved after the first run)")
    ap.add_argument("--key", help="your account key from the website (saved after the first run)")
    ap.add_argument("--dry-run", action="store_true", help="show what would be sent without sending anything")
    ap.add_argument("--every", type=int, default=300, help="watch: seconds between syncs (default 300)")
    ap.add_argument("--version", action="version", version=f"gitorio agent {AGENT_VERSION}")
    args = ap.parse_args()

    cfg = load_json(HOME / "config.json", {})
    if args.server: cfg["server"] = args.server
    if args.key: cfg["key"] = args.key
    if args.server or args.key:
        cfg["_unsaved"] = True      # written by run_sync after a successful connection, never before

    if args.command == "status":
        say(f"agent v{AGENT_VERSION}   home: {HOME}")
        say(f"server: {cfg.get('server', '(not set)')}   key: {(cfg.get('key') or '(not set)')[:9]}{'...' if cfg.get('key') else ''}")
        say(f"extra repositories: {', '.join(cfg.get('extra_repos', [])) or 'none'}")
        return 0
    if args.command == "add":
        if not args.path: sys.exit("usage: agent.py add <path-to-a-repository>")
        root = extract.git_root(args.path)
        if not root: sys.exit(f"{args.path} is not inside a git repository")
        cfg.setdefault("extra_repos", [])
        if root not in cfg["extra_repos"]: cfg["extra_repos"].append(root)
        cfg.pop("_unsaved", None)
        save_json(HOME / "config.json", cfg)
        say(f"Added {root}. Run the agent to report it to the website.")
        return 0

    if not cfg.get("server") or not cfg.get("key"):
        sys.exit("First run: python gitorio.py --server https://gitorio.com --key gtr_...   (create a key on the website's dashboard)")
    try:
        if args.command == "watch":
            while True:
                try:
                    run_sync(cfg)
                except ApiError as e:
                    say(f"! {e}")
                    if e.status in (401, 403): return 1
                say(f"Next sync in {args.every}s (Ctrl+C to stop)")
                time.sleep(args.every)
        return run_sync(cfg, dry_run=args.dry_run, force_report=False)
    except ApiError as e:
        say(f"! {e.message} ({e.code})")
        return 1
    except KeyboardInterrupt:
        say("stopped"); return 0


if __name__ == "__main__":
    sys.exit(main())
