Website & Codebase Reverse Engineering

# ⌁ GitReverse — Clone (live demo) A fully functional clone of **[gitreverse.com](https://www.gitreverse.com/)** — reverse-engineer any **GitHub codebase** or **website** into a single AI "build prompt" you can paste into Cursor / Claude Code / Codex. This notebook builds and runs the **live demo**: a self-contained **FastAPI** web app with a terminal-style reverse console. A faithful **Next.js / TypeScript** codebase is produced in parallel (packaged separately at the end). **Both input modes** - **Codebase** → GitHub API (metadata + root file tree + README) → LLM → one grounded build prompt. - **Website** → fetch + clean the page → LLM → a "rebuild this site" prompt. **LLM handling** — multi-provider (`OpenRouter → Groq → Google → OpenAI → xAI`, auto-picks the first key found) with a **deterministic offline synthesizer** so it works with **no API key at all**. **Cells below:** setup → LLM layer → pipelines → FastAPI server → web UI → launch & smoke test → live demo (real repo + real website) → package the Next.js codebase.

# Probe: does the sandbox have outbound internet from Python, and what do the key endpoints return?
import urllib.request, urllib.error, json, ssl

ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE

targets = [
    ("github_api", "https://api.github.com/repos/filiksyos/gitreverse"),
    ("robots",     "https://www.gitreverse.com/robots.txt"),
    ("sitemap",    "https://www.gitreverse.com/sitemap.xml"),
    ("library",    "https://www.gitreverse.com/library"),
]

def probe(name, url):
    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (compatible; gitreverse-extractor-probe)"})
    try:
        with urllib.request.urlopen(req, timeout=20, context=ctx) as r:
…

[OK ] github_api  200  len(head)=4000  ctype=application/json; charset=utf-8
      {"id":1197496497,"node_id":"R_kgDOR2BYsQ","name":"gitreverse","full_name":"filiksyos/gitreverse","private":false,"owner":{"login":"filiksyos","id":126945152,"node_id":"U_kgDOB5EHgA","avatar_url":"https://avatars.githubusercontent.com/u/126945152?v=4","gravatar_id":"","url":"https://api.github.com/us

[OK ] robots      200  len(head)=120  ctype=text/plain; charset=utf-8
      User-Agent: * Allow: / Disallow: /api/…

# Deep probe #1: full sitemap inventory + library page structure markers
import urllib.request, ssl, re, json
from collections import Counter

ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HDR = {"User-Agent":"Mozilla/5.0 (compatible; gitreverse-extractor-probe)"}

def get(url, timeout=40):
    req = urllib.request.Request(url, headers=HDR)
    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
        return r.read().decode("utf-8","replace"), dict(r.headers)

# --- sitemap ---
sm, hdr = get("https://gitreverse.com/sitemap.xml")
print("sitemap bytes:", len(sm), "ctype:", hdr.get("Content-Type"))
locs = re.findall(r"<loc>\s*(.*?)\s*</loc>", sm)
print("total <loc>:", len(locs))

…

sitemap bytes: 174679 ctype: application/xml
total <loc>: 1002

CATEGORY COUNTS:
     1000  code(owner/repo)
        1  home
        1  library

SAMPLES:
  [home] https://gitreverse.com
  [library] https://gitreverse.com/library
  [code(owner/repo)] https://gitreverse.com/muchaisam/Tours-Travel

website locs with ?url=: 0 of 0
has <urlset>: True

# Deep probe #2: /library page internals — RSC flight data, pagination mechanism, embedded items
import urllib.request, ssl, re
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HDR = {"User-Agent":"Mozilla/5.0 (compatible; gitreverse-extractor-probe)"}
def get(url, timeout=40):
    req = urllib.request.Request(url, headers=HDR)
    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
        return r.read().decode("utf-8","replace")

html = get("https://gitreverse.com/library")
print("library HTML bytes:", len(html))
print("has __NEXT_DATA__:", "__NEXT_DATA__" in html)
print("self.__next_f.push count:", html.count("self.__next_f.push"))
print("buildId match:", re.findall(r'"buildId"\s*:\s*"([^"]+)"', html)[:1])
print("dpl-id:", re.findall(r'data-dpl-id="([^"]+)"', html)[:1])

# 'Load more' / pagination markers
for kw in ["Load 24 more","Load more","load more","more","Next-Action","next-action","server action"]:
…

library HTML bytes: 85736
has __NEXT_DATA__: False
self.__next_f.push count: 9
buildId match: []
dpl-id: ['dpl_2jZVPRfeUEuJDYUcz7hjjpM3UxgL']
  kw 'Load 24 more': idx=-1
  kw 'Load more': idx=-1
  kw 'load more': idx=-1
  kw 'more': idx=60877
  kw 'Next-Action': idx=-1
  kw 'next-action': idx=-1
  kw 'server action': idx=-1

flight chunks: 9
decoded flight blob bytes: 20794
  blob has 'full_prompt': 0
  blob has 'prompt': 31
  blob has 'Reverse engineered': 0
  blob has 'target_url': 9
  blob ha…

# Deep probe #3: reconstruct list-item shape + test pagination params
import urllib.request, ssl, re
ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HDR = {"User-Agent":"Mozilla/5.0 (compatible; gitreverse-extractor-probe)"}
def get(url, timeout=40):
    req = urllib.request.Request(url, headers=HDR)
    with urllib.request.urlopen(req, timeout=timeout, context=ctx) as r:
        return r.read().decode("utf-8","replace")

def blob_of(url):
    html = get(url)
    chunks = re.findall(r'self\.__next_f\.push\(\[1,"((?:[^"\\]|\\.)*)"\]\)', html)
    return "".join(c.encode().decode("unicode_escape") for c in chunks), html

blob, html = blob_of("https://gitreverse.com/library")

def windows(term, n=2, w=350):
    out=[]; start=0
…

=== windows around 'target_url' ===
=== windows around '"owner"' ===
owner/repo pairs in page-1 blob: 0 []
  page=2           -> pairs=0 sig=d41d8cd98f sample=[]
  offset=24        -> pairs=0 sig=d41d8cd98f sample=[]
  sort=oldest      -> pairs=0 sig=d41d8cd98f sample=[]
  mode=website     -> pairs=0 sig=d41d8cd98f sample=[]

# Inspect the source repo for the library data source (method A)
import urllib.request, json, ssl, re

ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HDR = {"User-Agent":"gitreverse-extractor"}

RAW = "https://raw.githubusercontent.com/filiksyos/gitreverse/main"

def fetch(*paths):
    for p in paths:
        url = f"{RAW}/{p}"
        try:
            req = urllib.request.Request(url, headers=HDR)
            with urllib.request.urlopen(req, timeout=20, context=ctx) as r:
                body = r.read().decode("utf-8","replace")
            print(f"\n{'='*60}\n>>> {p}  ({len(body)} bytes)\n{'='*60}")
            print(body[:3000])
            if len(body) > 3000: print(f"... ({len(body)-3000} more bytes)")
…

============================================================
>>> .env.example  (3413 bytes)
============================================================
# Copy to .env.local and fill in values. See README for provider setup.

# --- Supabase (optional: response caching is off if these are empty) ---
SUPABASE_URL=
SUPABASE_PUBLISHABLE_KEY=

# --- GitHub (optional: unauthenticated requests have lower API rate limits) ---
GITHUB_TOKEN=

# --- Quick LLM (GET prompt / reverse-prompt API) ---…

# Find Supabase client init + tables in the source repo and live JS
import urllib.request, json, ssl, re

ctx = ssl.create_default_context(); ctx.check_hostname=False; ctx.verify_mode=ssl.CERT_NONE
HDR = {"User-Agent":"gitreverse-extractor"}

RAW = "https://raw.githubusercontent.com/filiksyos/gitreverse/main"

# 1. List lib/ and app/ directories from GitHub API
GITHUB_API = "https://api.github.com/repos/filiksyos/gitreverse/contents"
for subdir in ["lib", "app/library", "app", "components"]:
    url = f"{GITHUB_API}/{subdir}"
    try:
        req = urllib.request.Request(url, headers=HDR)
        with urllib.request.urlopen(req, timeout=15, context=ctx) as r:
            items = json.loads(r.read())
        print(f"\n=== {subdir}/ ===")
        for item in items:
…