mirror your GitHub repos to tangled.org automatically
1import { Buffer } from 'node:buffer'
2import { RemoteRejectedError, WireError } from './errors'
3import { encodePktLine, flushPkt, lineToString, PktLineReader } from './pkt-line'
4import { type Advertisement, parseAdvertisement } from './refs'
5
6const AGENT = 'synchub.to'
7const ADVERTISEMENT_TIMEOUT_MS = 30_000
8
9function repoUrl(repoFullName: string): string {
10 return `https://github.com/${repoFullName}.git`
11}
12
13function authHeader(token: string): string {
14 return `Basic ${Buffer.from(`x-access-token:${token}`).toString('base64')}`
15}
16
17async function* streamBytes(body: ReadableStream<Uint8Array>): AsyncGenerator<Buffer> {
18 const reader = body.getReader()
19 try {
20 for (;;) {
21 // eslint-disable-next-line no-await-in-loop -- sequential drain of the response body
22 const { value, done } = await reader.read()
23 if (done) return
24 if (value) yield Buffer.from(value)
25 }
26 }
27 finally {
28 reader.releaseLock()
29 }
30}
31
32/**
33 * Fetch GitHub's `git-upload-pack` ref advertisement over smart HTTP. We need
34 * this both to resolve a ref name to a SHA (create-ref path) and, more
35 * generally, to learn the capability set before negotiating.
36 */
37export async function fetchAdvertisement(repoFullName: string, token: string): Promise<Advertisement> {
38 const url = `${repoUrl(repoFullName)}/info/refs?service=git-upload-pack`
39 const res = await fetch(url, {
40 headers: {
41 Authorization: authHeader(token),
42 // Pin protocol v0; v2 would frame the advertisement differently.
43 'Git-Protocol': 'version=0',
44 },
45 signal: AbortSignal.timeout(ADVERTISEMENT_TIMEOUT_MS),
46 })
47 if (!res.ok || !res.body) {
48 throw new WireError(`github info/refs failed: ${res.status} ${res.statusText}`)
49 }
50 const reader = new PktLineReader(streamBytes(res.body))
51 const lines = await reader.readUntilFlush()
52 // The first flush ends the `# service` banner; the advertisement follows.
53 const adv = await reader.readUntilFlush()
54 return parseAdvertisement([...(lines ?? []), ...(adv ?? [])])
55}
56
57export interface FetchPackOptions {
58 repoFullName: string
59 token: string
60 /** SHA we want fetched. Requires GitHub's allow-reachable-sha1-in-want. */
61 want: string
62 /** Knot ref tips to advertise as haves so GitHub sends a thin delta. */
63 haves: string[]
64 /** Abort and throw `too-big` once the pack exceeds this many bytes. */
65 maxBytes: number
66}
67
68export interface FetchPackResult {
69 /** Raw packfile bytes. Pipe straight into receive-pack; do not buffer. */
70 pack: AsyncGenerator<Buffer>
71}
72
73/**
74 * Negotiate a thin pack from GitHub for `want`, advertising `haves` so the
75 * server deltas against objects the knot already holds. Returns a streaming
76 * generator of the raw packfile bytes; the caller pipes them into
77 * receive-pack and never materialises them.
78 *
79 * Protocol v0, no side-band: after the single NAK/ACK pkt-line the response
80 * body is the raw packfile to EOF, which is exactly what we forward.
81 */
82export async function fetchPack(opts: FetchPackOptions): Promise<FetchPackResult> {
83 const { repoFullName, token, want, haves, maxBytes } = opts
84
85 const wantLine = `want ${want} thin-pack ofs-delta agent=${AGENT}/1\n`
86 const body: Buffer[] = [encodePktLine(wantLine), flushPkt]
87 for (const have of haves) {
88 body.push(encodePktLine(`have ${have}\n`))
89 }
90 body.push(encodePktLine('done\n'))
91
92 const res = await fetch(`${repoUrl(repoFullName)}/git-upload-pack`, {
93 method: 'POST',
94 headers: {
95 Authorization: authHeader(token),
96 'Content-Type': 'application/x-git-upload-pack-request',
97 'Accept': 'application/x-git-upload-pack-result',
98 'Git-Protocol': 'version=0',
99 },
100 body: Buffer.concat(body),
101 })
102 if (!res.ok || !res.body) {
103 throw new WireError(`github git-upload-pack failed: ${res.status} ${res.statusText}`)
104 }
105
106 const reader = new PktLineReader(streamBytes(res.body))
107 // Read the negotiation result: one ACK/NAK line, or an ERR line on failure.
108 const ack = await reader.next()
109 if (ack === null || ack.type === 'flush') {
110 throw new WireError('github git-upload-pack: empty negotiation response')
111 }
112 const ackStr = lineToString(ack.data)
113 if (ackStr.startsWith('ERR ')) {
114 // `ERR upload-pack: not our ref` is a propagation race on GitHub's side;
115 // surface as a plain WireError so the queue retries with backoff.
116 throw new WireError(`github git-upload-pack: ${ackStr.slice(4)}`)
117 }
118 if (!ackStr.startsWith('ACK') && !ackStr.startsWith('NAK')) {
119 throw new WireError(`github git-upload-pack: unexpected negotiation line ${JSON.stringify(ackStr)}`)
120 }
121
122 async function* capped(): AsyncGenerator<Buffer> {
123 let total = 0
124 for await (const chunk of reader.remaining()) {
125 total += chunk.length
126 if (total > maxBytes) {
127 throw new RemoteRejectedError(`pack exceeded ${maxBytes} bytes`, 'too-big')
128 }
129 yield chunk
130 }
131 }
132
133 return { pack: capped() }
134}