316 lines
15 KiB
Python
316 lines
15 KiB
Python
"""Bounded issue reads and explicit stage-label writes; remote bodies are untrusted."""
|
|
import base64
|
|
from http.client import HTTPException
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import ssl
|
|
from time import monotonic
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode, urlsplit
|
|
from urllib.request import build_opener, HTTPRedirectHandler, HTTPSHandler, Request
|
|
|
|
from .transaction import InstallError, safe_path
|
|
|
|
|
|
def _url(value, ssh=False):
|
|
if (not isinstance(value, str) or not value or not value.isascii()
|
|
or any(c.isspace() or ord(c) < 32 for c in value)
|
|
or any(c in value for c in ('\\', '?', '#', '%'))):
|
|
raise InstallError('invalid repository URL')
|
|
if ssh and value.startswith('git@'):
|
|
match = re.fullmatch(r'git@([^/:]+):(.+)', value)
|
|
if not match:
|
|
raise InstallError('invalid repository URL')
|
|
value = 'ssh://git@' + match[1] + '/' + match[2]
|
|
try:
|
|
parsed = urlsplit(value)
|
|
allowed = ('https', 'ssh') if ssh else ('https',)
|
|
if parsed.scheme not in allowed or not parsed.hostname:
|
|
raise ValueError()
|
|
if parsed.password is not None or parsed.username != ('git' if parsed.scheme == 'ssh' else None):
|
|
raise ValueError()
|
|
host = parsed.hostname.lower()
|
|
if not re.fullmatch(r'[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?', host) or '..' in host:
|
|
raise ValueError()
|
|
port = parsed.port
|
|
if parsed.netloc.endswith(':') or (port is not None and port == 0):
|
|
raise ValueError()
|
|
if parsed.scheme == 'ssh' and port not in (None, 22):
|
|
raise ValueError()
|
|
authority = host
|
|
if parsed.scheme == 'https' and port not in (None, 443):
|
|
authority += ':' + str(port)
|
|
path = parsed.path.rstrip('/')
|
|
if path and (not path.startswith('/') or any(
|
|
not re.fullmatch(r'[A-Za-z0-9_.-]+', part) or part in ('.', '..')
|
|
for part in path[1:].split('/'))):
|
|
raise ValueError()
|
|
return 'https://' + authority, path
|
|
except ValueError:
|
|
raise InstallError('invalid repository URL') from None
|
|
|
|
|
|
def canonical_repository(url):
|
|
"""Normalize credential-free HTTPS and conventional Git SSH remote URLs."""
|
|
origin, path = _url(url, ssh=True)
|
|
if path.endswith('.git'):
|
|
path = path[:-4]
|
|
if len(path.split('/')) < 3 or not path.split('/')[-1] or path.split('/')[-1] in ('.', '..'):
|
|
raise InstallError('repository URL requires owner and repository')
|
|
return origin + path
|
|
|
|
|
|
MAX_PAGES = 100
|
|
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
|
MAX_TOTAL_BYTES = 16 * 1024 * 1024
|
|
MAX_CREDENTIAL_BYTES = 64 * 1024
|
|
REQUEST_TIMEOUT = 10
|
|
OPERATION_TIMEOUT = 60
|
|
PAGE_SIZE = 100
|
|
STAGE_LABELS = frozenset(('ai-ready', 'ai-in-progress', 'ai-testing', 'ai-waiting-user',
|
|
'ai-ready-to-merge', 'ai-merging', 'ai-done', 'ai-blocked'))
|
|
|
|
|
|
class TrackerAccessDenied(InstallError):
|
|
"""Authentication/permission denial needs explicit reconciliation."""
|
|
pass
|
|
|
|
|
|
class TrackerHTTPFailure(InstallError):
|
|
def __init__(self, status):
|
|
super().__init__('issue tracker HTTP request failed')
|
|
self.status = status
|
|
|
|
|
|
class NoRedirect(HTTPRedirectHandler):
|
|
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
return None
|
|
|
|
|
|
def _credential_text(value):
|
|
if (not isinstance(value, str) or not value or not value.isascii()
|
|
or any(ord(c) < 32 or ord(c) == 127 for c in value)):
|
|
raise InstallError('invalid tracker authentication')
|
|
return value
|
|
|
|
|
|
class Tracker:
|
|
"""Access one configured repository; credentials never follow remote-provided links."""
|
|
|
|
def __init__(self, kind, repository, root: Path, credentials_path=None, before_write=None):
|
|
if kind not in ('gitea', 'github'):
|
|
raise InstallError('unsupported issue tracker')
|
|
self.kind = kind
|
|
self.before_write = before_write
|
|
self.repository = canonical_repository(repository)
|
|
origin, path = _url(self.repository)
|
|
parts = path[1:].split('/')
|
|
if kind == 'github':
|
|
if origin != 'https://github.com' or len(parts) != 2:
|
|
raise InstallError('only public hosted GitHub repositories are supported')
|
|
self.service = 'https://github.com'
|
|
self.api = 'https://api.github.com/repos/' + '/'.join(parts)
|
|
else:
|
|
self.service = origin + ('/' + '/'.join(parts[:-2]) if len(parts) > 2 else '')
|
|
self.api = self.service + '/api/v1/repos/' + '/'.join(parts[-2:])
|
|
self._authorization = self._credentials(Path(root), credentials_path)
|
|
|
|
def _token(self, token):
|
|
token = _credential_text(token)
|
|
if any(c.isspace() for c in token):
|
|
raise InstallError('invalid tracker authentication')
|
|
return ('Bearer ' if self.kind == 'github' else 'token ') + token
|
|
|
|
def _credentials(self, root, credentials_path):
|
|
token = os.environ.get('PDA_' + self.kind.upper() + '_TOKEN')
|
|
if token:
|
|
return self._token(token)
|
|
try:
|
|
relative = Path(credentials_path or '.local/secrets/access.json')
|
|
if relative.is_absolute():
|
|
relative = relative.relative_to(root.resolve())
|
|
path = safe_path(root, relative.as_posix())
|
|
if not path.exists():
|
|
if credentials_path is not None:
|
|
raise ValueError()
|
|
return None
|
|
with path.open('rb') as handle:
|
|
raw = handle.read(MAX_CREDENTIAL_BYTES + 1)
|
|
if len(raw) > MAX_CREDENTIAL_BYTES:
|
|
raise ValueError()
|
|
value = json.loads(raw)
|
|
if not isinstance(value, dict) or not isinstance(value.get('services'), dict):
|
|
raise ValueError()
|
|
service = value['services'].get(self.kind)
|
|
if service is None:
|
|
return None
|
|
if not isinstance(service, dict):
|
|
raise ValueError()
|
|
if all(service.get(key) in (None, '') for key in ('token', 'username', 'password')):
|
|
return None
|
|
origin, path = _url(service.get('url'))
|
|
allowed = {self.service}
|
|
if self.kind == 'github':
|
|
allowed.add('https://api.github.com')
|
|
if origin + path not in allowed:
|
|
raise ValueError()
|
|
if service.get('token') not in (None, ''):
|
|
return self._token(service['token'])
|
|
if 'username' in service or 'password' in service:
|
|
if self.kind != 'gitea':
|
|
raise ValueError()
|
|
username = _credential_text(service.get('username'))
|
|
password = _credential_text(service.get('password'))
|
|
if ':' in username:
|
|
raise ValueError()
|
|
return 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode('ascii')
|
|
return None
|
|
except (OSError, ValueError, TypeError, UnicodeError, RecursionError):
|
|
raise InstallError('invalid tracker authentication configuration') from None
|
|
|
|
def _budget(self):
|
|
return [monotonic() + OPERATION_TIMEOUT, 0]
|
|
|
|
def _remaining(self, budget):
|
|
remaining = budget[0] - monotonic()
|
|
if remaining <= 0:
|
|
raise InstallError('issue tracker operation time limit exceeded')
|
|
return min(REQUEST_TIMEOUT, remaining)
|
|
|
|
def _request(self, suffix, budget, query=None, *, method='GET', data=None, success=(200,)):
|
|
url = self.api + suffix
|
|
if query:
|
|
url += '?' + urlencode(query)
|
|
headers = {'Accept': 'application/json', 'User-Agent': 'personal-dev-agent'}
|
|
if self.kind == 'github':
|
|
headers.update({'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'})
|
|
if self._authorization:
|
|
headers['Authorization'] = self._authorization
|
|
if data is not None:
|
|
headers['Content-Type'] = 'application/json'
|
|
request = Request(url, headers=headers, method=method,
|
|
data=json.dumps(data).encode() if data is not None else None)
|
|
try:
|
|
opener = build_opener(NoRedirect(), HTTPSHandler(context=ssl.create_default_context()))
|
|
timeout = self._remaining(budget)
|
|
# Recheck local authorization after all prior remote reads, at the write boundary.
|
|
if request.get_method().upper() not in ('GET', 'HEAD') and self.before_write is not None:
|
|
self.before_write()
|
|
with opener.open(request, timeout=timeout) as response:
|
|
if response.status not in success:
|
|
raise InstallError('issue tracker returned an unsuccessful status')
|
|
length = response.headers.get('Content-Length')
|
|
if length is not None and int(length) > MAX_RESPONSE_BYTES:
|
|
raise InstallError('issue tracker response size limit exceeded')
|
|
chunks, size = [], 0
|
|
while True:
|
|
self._remaining(budget)
|
|
chunk = response.read1(min(65536, MAX_RESPONSE_BYTES - size + 1))
|
|
self._remaining(budget)
|
|
if not chunk:
|
|
break
|
|
size += len(chunk)
|
|
budget[1] += len(chunk)
|
|
if size > MAX_RESPONSE_BYTES or budget[1] > MAX_TOTAL_BYTES:
|
|
raise InstallError('issue tracker response size limit exceeded')
|
|
chunks.append(chunk)
|
|
raw = b''.join(chunks)
|
|
return None if response.status == 204 and not raw else json.loads(raw)
|
|
except InstallError:
|
|
raise
|
|
except HTTPError as error:
|
|
error.close()
|
|
if error.code in (401, 403):
|
|
raise TrackerAccessDenied('issue tracker access denied; verify authentication and permission') from None
|
|
raise TrackerHTTPFailure(error.code) from None
|
|
except (OSError, URLError, HTTPException, ValueError, UnicodeError, RecursionError):
|
|
raise InstallError('issue tracker request or response failed') from None
|
|
|
|
def _pages(self, suffix, budget, state=None):
|
|
records = []
|
|
for page in range(1, MAX_PAGES + 1):
|
|
query = {'page': page, 'limit' if self.kind == 'gitea' else 'per_page': PAGE_SIZE}
|
|
if state:
|
|
query['state'] = state
|
|
value = self._request(suffix, budget, query)
|
|
if not isinstance(value, list) or any(not isinstance(item, dict) for item in value):
|
|
raise InstallError('invalid issue tracker list response')
|
|
if not value:
|
|
return records
|
|
records.extend(value)
|
|
raise InstallError('issue tracker pagination limit exceeded')
|
|
|
|
def issues(self):
|
|
"""Return open issues; Gitea may include pull_request: null on them."""
|
|
return [item for item in self._pages('/issues', self._budget(), state='open')
|
|
if item.get('pull_request') is None and item.get('state', 'open') == 'open']
|
|
|
|
def _labels(self, suffix, budget):
|
|
# Gitea paginates repository labels, but not the labels attached to an issue.
|
|
labels = (self._request(suffix, budget) if self.kind == 'gitea' and suffix.startswith('/issues/')
|
|
else self._pages(suffix, budget))
|
|
if not isinstance(labels, list) or any(
|
|
not isinstance(item, dict) or not isinstance(item.get('name'), str) or not item['name']
|
|
or type(item.get('id')) is not int or item['id'] <= 0 for item in labels):
|
|
raise InstallError('invalid issue tracker label response')
|
|
return labels
|
|
|
|
def set_stage(self, number, label):
|
|
"""Synchronize presentation only; caller must already own the remote issue claim."""
|
|
if type(number) is not int or number <= 0:
|
|
raise InstallError('issue number must be a positive integer')
|
|
if not isinstance(label, str) or label not in STAGE_LABELS:
|
|
raise InstallError('unsupported issue stage label')
|
|
budget = self._budget()
|
|
suffix = '/issues/' + str(number)
|
|
issue = self._request(suffix, budget)
|
|
if (not isinstance(issue, dict) or issue.get('number') != number
|
|
or issue.get('pull_request') is not None):
|
|
raise InstallError('invalid issue tracker issue response')
|
|
available = self._labels('/labels', budget)
|
|
target = [item for item in available if item['name'] == label]
|
|
if not target:
|
|
try:
|
|
created = self._request('/labels', budget, method='POST', success=(201,), data={
|
|
'name': label, 'color': '#1d76db' if self.kind == 'gitea' else '1d76db'})
|
|
except TrackerHTTPFailure as error:
|
|
# Two different issue owners may create the same repository label.
|
|
# Only a definite conflict can be reconciled; timeout/denial still fails.
|
|
if error.status not in (409, 422):
|
|
raise
|
|
target = [item for item in self._labels('/labels', budget) if item['name'] == label]
|
|
else:
|
|
target = [created]
|
|
if (len(target) != 1 or not isinstance(target[0], dict) or target[0].get('name') != label
|
|
or type(target[0].get('id')) is not int or target[0]['id'] <= 0):
|
|
raise InstallError('issue tracker stage label creation not confirmed')
|
|
current = self._labels(suffix + '/labels', budget)
|
|
if not any(item['name'] == label for item in current):
|
|
value = target[0]['id'] if self.kind == 'gitea' else label
|
|
self._request(suffix + '/labels', budget, method='POST', data={'labels': [value]})
|
|
for item in current:
|
|
if item['name'] in STAGE_LABELS and item['name'] != label:
|
|
value = str(item['id']) if self.kind == 'gitea' else item['name']
|
|
self._request(suffix + '/labels/' + value, budget, method='DELETE',
|
|
success=(204,) if self.kind == 'gitea' else (200,))
|
|
stages = [item['name'] for item in self._labels(suffix + '/labels', budget)
|
|
if item['name'] in STAGE_LABELS]
|
|
if stages != [label]:
|
|
raise InstallError('issue tracker stage label readback mismatch; reconcile before continuing')
|
|
return {'number': number, 'label': label}
|
|
|
|
def issue(self, number):
|
|
"""Return the issue and its complete comments_data list as untrusted data."""
|
|
if type(number) is not int or number <= 0:
|
|
raise InstallError('issue number must be a positive integer')
|
|
budget = self._budget()
|
|
suffix = '/issues/' + str(number)
|
|
value = self._request(suffix, budget)
|
|
if not isinstance(value, dict) or value.get('number') != number or value.get('pull_request') is not None:
|
|
raise InstallError('invalid issue tracker issue response')
|
|
result = dict(value)
|
|
result['comments_data'] = self._pages(suffix + '/comments', budget)
|
|
return result
|