158 lines
7.0 KiB
Python
158 lines
7.0 KiB
Python
"""Read one fixed public GitHub source; verify data without executing its code."""
|
|
import ast
|
|
from contextlib import contextmanager
|
|
import hashlib
|
|
import io
|
|
import json
|
|
from pathlib import Path, PurePosixPath
|
|
import tarfile
|
|
import tempfile
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from .source import CHANNEL, PROTOCOL, REPOSITORY, RUNTIME, STATIC, revision
|
|
from .transaction import InstallError
|
|
|
|
API = 'https://api.github.com/repos/' + REPOSITORY
|
|
MAX_FILE = 1024 * 1024
|
|
MAX_TOTAL = 12 * MAX_FILE
|
|
|
|
|
|
class Redirects(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
parsed = urllib.parse.urlparse(newurl)
|
|
if parsed.scheme != 'https' or parsed.hostname not in ('api.github.com', 'codeload.github.com'):
|
|
raise InstallError('unexpected download redirect')
|
|
return super().redirect_request(req, fp, code, msg, headers, newurl)
|
|
|
|
|
|
class Client:
|
|
def __init__(self):
|
|
self.deadline = time.monotonic() + 45
|
|
|
|
def get(self, path, binary=False):
|
|
remaining = self.deadline - time.monotonic()
|
|
if remaining <= 0:
|
|
raise InstallError('GitHub request deadline exceeded')
|
|
request = urllib.request.Request(API + path, headers={
|
|
'Accept': 'application/vnd.github+json', 'User-Agent': 'personal-dev-updater',
|
|
'X-GitHub-Api-Version': '2022-11-28'})
|
|
try:
|
|
with urllib.request.build_opener(Redirects()).open(request, timeout=min(8, remaining)) as response:
|
|
chunks, size = [], 0
|
|
while True:
|
|
if time.monotonic() > self.deadline:
|
|
raise InstallError('GitHub request deadline exceeded')
|
|
chunk = response.read(65536)
|
|
if not chunk:
|
|
break
|
|
size += len(chunk)
|
|
if size > MAX_TOTAL:
|
|
raise InstallError('GitHub response exceeds size limit')
|
|
chunks.append(chunk)
|
|
raw = b''.join(chunks)
|
|
return raw if binary else json.loads(raw)
|
|
except (OSError, ValueError, urllib.error.URLError) as exc:
|
|
# Do not echo response bodies, credentials or local proxy details.
|
|
raise InstallError('GitHub unavailable or invalid response (' + type(exc).__name__ + ')') from exc
|
|
|
|
|
|
def latest():
|
|
return revision(Client().get('/git/ref/heads/' + CHANNEL)['object']['sha'])
|
|
|
|
|
|
def compare(base, target):
|
|
base, target = revision(base), revision(target)
|
|
value = Client().get('/compare/' + base + '...' + target + '?per_page=20')
|
|
status = value.get('status')
|
|
if status not in ('ahead', 'behind', 'identical', 'diverged'):
|
|
raise InstallError('invalid GitHub comparison')
|
|
summary = []
|
|
for item in value.get('commits', [])[:20]:
|
|
line = item['commit']['message'].splitlines()[0][:240]
|
|
summary.append(''.join(c for c in line if c.isprintable()))
|
|
return {'status': status, 'summary': summary,
|
|
'url': 'https://github.com/' + REPOSITORY + '/compare/' + base + '...' + target}
|
|
|
|
|
|
def selected(name):
|
|
return name in RUNTIME or name in STATIC or (
|
|
name.startswith('skills/personal-dev/') and name.endswith('.md'))
|
|
|
|
|
|
def validate_path(name):
|
|
path = PurePosixPath(name)
|
|
if (not name or '\\' in name or path.is_absolute() or
|
|
any(part in ('', '.', '..') for part in name.split('/'))):
|
|
raise InstallError('invalid package path')
|
|
|
|
|
|
@contextmanager
|
|
def download(commit):
|
|
commit = revision(commit)
|
|
client = Client()
|
|
tree = client.get('/git/trees/' + commit + '?recursive=1')
|
|
if tree.get('truncated'):
|
|
raise InstallError('source tree is truncated')
|
|
entries = {}
|
|
for item in tree['tree']:
|
|
name = item['path']
|
|
if name.startswith('scripts/pda/') and name.endswith('.py') and name not in RUNTIME:
|
|
raise InstallError('unsupported runtime layout; use a reviewed full installer')
|
|
if not selected(name):
|
|
continue
|
|
validate_path(name)
|
|
if item.get('mode') not in ('100644', '100755') or item['type'] != 'blob':
|
|
raise InstallError('package contains non-regular files')
|
|
if name in entries or item.get('size', MAX_FILE + 1) > MAX_FILE:
|
|
raise InstallError('invalid package entry or size')
|
|
revision(item['sha'])
|
|
entries[name] = item
|
|
required = set(RUNTIME + STATIC) | {'skills/personal-dev/SKILL.md'}
|
|
if not required.issubset(entries) or len(entries) > 200:
|
|
raise InstallError('source lacks supported updater payload; use a reviewed full installer')
|
|
raw = client.get('/tarball/' + commit, binary=True)
|
|
with tempfile.TemporaryDirectory(prefix='pda-update-') as directory:
|
|
root = Path(directory)
|
|
found, total, archive_total, member_count = set(), 0, 0, 0
|
|
try:
|
|
with tarfile.open(fileobj=io.BytesIO(raw), mode='r|gz') as archive:
|
|
for member in archive:
|
|
archive_total += member.size
|
|
member_count += 1
|
|
if member.size < 0 or archive_total > MAX_TOTAL or member_count > 5000:
|
|
raise InstallError('archive exceeds size limit')
|
|
if time.monotonic() > client.deadline:
|
|
raise InstallError('package extraction deadline exceeded')
|
|
parts = member.name.split('/', 1)
|
|
if len(parts) != 2 or parts[1] not in entries:
|
|
continue
|
|
name = parts[1]
|
|
validate_path(member.name)
|
|
item = entries[name]
|
|
if (name in found or not member.isfile() or member.size != item['size']
|
|
or member.size > MAX_FILE):
|
|
raise InstallError('invalid archive entry')
|
|
total += member.size
|
|
if total > MAX_TOTAL:
|
|
raise InstallError('package exceeds size limit')
|
|
data = archive.extractfile(member).read(MAX_FILE + 1)
|
|
digest = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
|
|
if digest != item['sha']:
|
|
raise InstallError('package integrity mismatch')
|
|
if name.endswith('.py'):
|
|
ast.parse(data, filename=name) # Syntax check only; never execute.
|
|
target = root / name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(data)
|
|
found.add(name)
|
|
if found != set(entries):
|
|
raise InstallError('incomplete package archive')
|
|
if json.loads((root / 'profiles/update.json').read_bytes()) != {'protocol': PROTOCOL}:
|
|
raise InstallError('unsupported update protocol; use a reviewed full installer')
|
|
except (tarfile.TarError, SyntaxError, UnicodeError, EOFError) as exc:
|
|
raise InstallError('invalid package archive or runtime') from exc
|
|
yield root
|