github.py
Rama master · réplica de lectura del repositorio original.
"""GitHub destination — write each echo as a markdown file into a repository.
One connected account is one target repository (owner/repo + branch). Each
echo renders its template as the file body and writes it to a per-echo path
template, so a site, microblog, or link log lands in a repo as markdown.
Parity target is EchoFeed's GitHub service: token + repo + branch +
committer, with file path and commit message defined per echo.
Security:
- Fixed host ``https://api.github.com`` — no user-supplied base URL, so no
new SSRF surface. All calls are sync ``httpx`` with
``DEFAULT_REQUEST_TIMEOUT``.
- The token is a credential: encrypted at rest by the caller
(``security.encrypt_secret``), never rendered back in the UI, never
interpolated into errors or logs. Error strings are static.
- Path traversal is rejected, not silently repaired: a path that reaches a
``..`` came from a bad template and must fail loudly.
Error mapping: 401, and 403 that is not a rate limit, are
``GitHubAuthError`` (permanent); 404 repo/branch is ``GitHubNotFoundError``
(permanent); 403/429 with ``x-ratelimit-remaining: 0`` or a Retry-After is
``GitHubRateLimitError`` (transient); 409/422 is ``GitHubRejectedError``
(permanent); every other 5xx/3xx is ``GitHubError`` (transient).
"""
from __future__ import annotations
import base64
import logging
import re
from typing import Any, Callable
from urllib.parse import quote
import httpx
import template_engine
from feed_parser import SSRFError, pinned_request
from utils import (
DEFAULT_REQUEST_TIMEOUT,
DestinationAuthError,
DestinationError,
DestinationNotFoundError,
DestinationRateLimitError,
parse_retry_after,
)
logger = logging.getLogger(__name__)
REQUEST_TIMEOUT = DEFAULT_REQUEST_TIMEOUT
API_BASE = "https://api.github.com"
# Encoded-body cap: over the cap is a PERMANENT rejection — truncating a
# backup file silently corrupts it, so the item gives up instead.
MAX_FILE_BYTES = 1_000_000
# A path template renders to a repo-relative path: per-segment 255 chars,
# total 1024. Generous for real blogs, tight enough to stop abuse.
MAX_PATH_TOTAL = 1024
MAX_PATH_SEGMENT = 255
MAX_COMMIT_MESSAGE_CHARS = 200
DEFAULT_COMMIT_TEMPLATE = "Add {{ title }}"
# Sample item for save-time echo validation: deliberately adversarial so a
# naive `posts/{{ title }}.md` path fails AT SAVE TIME with a pointer to the
# `| slug` filter instead of permanently failing on the first real item. The
# title carries a newline (a control character normalize_path rejects), a
# slash and a colon (legal path separators / illegal filename characters),
# a `..` segment, and a 200-char tail. The tail length is deliberate: it must
# keep `{{ title | slug }}` (capped at 255 chars) plus the member's
# `_{{ date_iso }}.md` suffix inside the 255-char segment limit, so the
# recommended templates still validate against this sample.
SAMPLE_ITEM: dict = {
"id": "sample-item",
"title": "Feed\nEcho: test/demo .. post " + "x" * 200,
"link": "https://example.com/posts/1",
"summary": "Line one\nLine two",
"content": "",
"content_link": "",
"author": "",
"date": "2026-01-15T09:30:00",
"tags": ["news"],
"image_url": "",
"image_alt": "",
}
class GitHubError(DestinationError):
"""Base error for GitHub delivery (transient unless a subclass says so)."""
class GitHubRateLimitError(GitHubError, DestinationRateLimitError):
"""403/429 rate limit. Transient; carries the server's suggested wait."""
def __init__(self, retry_after: float | None = None):
self.retry_after = retry_after
msg = "GitHub rate limit hit"
if retry_after is not None:
msg += f" (retry after {retry_after:.0f}s)"
super().__init__(msg)
class GitHubAuthError(GitHubError, DestinationAuthError):
"""401, or 403 that is not a rate limit — token rejected or lacks the
Contents: Read and write permission."""
class GitHubNotFoundError(GitHubError, DestinationNotFoundError):
"""404 — the repository or branch no longer exists (or the token cannot
see it)."""
class GitHubRejectedError(GitHubError):
"""409/422 — GitHub refused the write (protected branch, invalid payload).
Permanent: retrying the same request cannot change the outcome."""
def normalize_token(raw: str) -> str:
"""Validate a GitHub token and return it stripped.
Shape checks only (length, no whitespace) — the live ``connect()`` call
is the real credential check. Raises ValueError with a user-facing
message.
"""
value = (raw or "").strip()
if not value:
raise ValueError("Enter a GitHub personal access token.")
if len(value) < 20:
raise ValueError("GitHub tokens are at least 20 characters.")
if len(value) > 200:
raise ValueError("GitHub tokens are at most 200 characters.")
if any(ch.isspace() for ch in value):
raise ValueError("GitHub tokens cannot contain whitespace.")
return value
def parse_repo(raw: str) -> str:
"""Normalize a repository reference to lowercase ``owner/repo``.
Accepts ``owner/repo``, ``https://github.com/owner/repo``, with or
without a ``.git`` suffix or trailing slash. Rejects anything else with
a user-facing ValueError.
"""
value = (raw or "").strip()
if not value:
raise ValueError("Enter the repository as owner/repo.")
lowered = value.lower()
for prefix in (
"https://github.com/", "http://github.com/", "github.com/",
):
if lowered.startswith(prefix):
value = value[len(prefix):]
break
else:
if lowered.startswith("git@github.com:"):
value = value[len("git@github.com:"):]
value = value.strip("/")
if value.lower().endswith(".git"):
value = value[:-4].strip("/")
parts = value.split("/")
if (
len(parts) != 2
or not parts[0]
or not parts[1]
or any(p in (".", "..") or p.startswith(".") for p in parts)
or any(any(c.isspace() for c in p) for p in parts)
or any(any(c in p for c in "?#:@%") for p in parts)
):
raise ValueError(
"Repository must look like owner/repo"
" (or the https://github.com/owner/repo URL)."
)
return "/".join(parts).lower()
def normalize_path(rendered: str) -> str:
"""Validate a rendered repo-relative path and return the stored form.
Normalizes ``\\`` to ``/``, strips every segment, and drops empty
segments (so ``"posts/"`` becomes ``"posts.md"`` rather than
``"posts/.md"``). Rejects (rather than silently repairs) an empty
result, an absolute path, ``.``/``..`` segments, control characters, a
segment over 255 chars, or a total over 1024 chars. Appends ``.md`` when
the last segment has no extension.
"""
if not rendered or not rendered.strip():
raise ValueError("GitHub file path is empty.")
if any(ord(c) < 32 or ord(c) == 127 for c in rendered):
raise ValueError("GitHub file path contains a control character.")
path = rendered.replace("\\", "/")
if path.startswith("/"):
raise ValueError("GitHub file path must be relative, not absolute.")
segments = [seg.strip() for seg in path.split("/")]
segments = [seg for seg in segments if seg]
if not segments:
raise ValueError("GitHub file path is empty.")
for seg in segments:
if seg in (".", ".."):
raise ValueError(
"GitHub file path must not contain . or .. segments."
)
if len(seg) > MAX_PATH_SEGMENT:
raise ValueError(
f"GitHub file path segment is over {MAX_PATH_SEGMENT} characters."
)
path = "/".join(segments)
if len(path) > MAX_PATH_TOTAL:
raise ValueError(
f"GitHub file path is over {MAX_PATH_TOTAL} characters."
)
last = segments[-1]
if "." not in last:
path += ".md"
return path
def render_path(template: str, item: Any, feed_name: str = "") -> str:
"""Render a path template for one item and validate the result.
``item`` is the flat item dict the scheduler passes (a FeedItem maps
fine). Raises template_engine exceptions on render failure and
ValueError on path validation failure — both are permanent at dispatch.
"""
rendered = template_engine.render_template(template, item, feed_name=feed_name)
return normalize_path(rendered)
def build_commit_message(
template: str, item: Any, feed_name: str = "", path: str = ""
) -> str:
"""Render the commit-message template, collapse whitespace, cap at 200.
A blank stored template falls back to ``Add {{ title }}``; a render that
comes out empty falls back to the path, so a commit always has a body.
"""
tpl = (template or "").strip() or DEFAULT_COMMIT_TEMPLATE
try:
rendered = template_engine.render_template(tpl, item, feed_name=feed_name)
except Exception:
# A stored commit template can drift out of sync with the engine
# (or the user hand-edited the DB). A commit message is decoration,
# not content — fall back to the path rather than fail the delivery.
rendered = ""
rendered = re.sub(r"\s+", " ", rendered).strip()
if not rendered:
rendered = re.sub(r"\s+", " ", path).strip() or "Update file"
return rendered[:MAX_COMMIT_MESSAGE_CHARS]
def _encode_content(content: str) -> str:
"""base64-encode the file body, enforcing the size cap BEFORE any network
call. Over the cap is a permanent rejection."""
encoded = base64.b64encode(content.encode("utf-8")).decode("ascii")
if len(encoded) > MAX_FILE_BYTES:
raise GitHubRejectedError(
f"File is over the {MAX_FILE_BYTES}-byte GitHub size cap"
" and cannot be delivered without truncation."
)
return encoded
def _quote_path(path: str) -> str:
"""Percent-encode a repo-relative path per segment.
``quote(path, safe='/')`` is not enough for ``#``/``?`` — those would
start a fragment/query inside the URL — so each segment is encoded with
an empty safe set and the slashes are re-joined.
"""
return "/".join(quote(seg, safe="") for seg in path.split("/"))
def _rate_limit_error(response) -> GitHubRateLimitError | None:
"""Map a 403/429 to a rate-limit error, or None when it is a real auth
refusal (403 with budget remaining and no Retry-After)."""
remaining = response.headers.get("x-ratelimit-remaining")
if remaining is not None:
try:
if int(remaining) == 0:
return GitHubRateLimitError(parse_retry_after(response))
except ValueError:
pass
retry_after = parse_retry_after(response)
if retry_after is not None:
return GitHubRateLimitError(retry_after)
if response.status_code == 429:
return GitHubRateLimitError(None)
return None
def _request(method: str, url: str, **kwargs) -> httpx.Response:
"""One pinned-transport request to the fixed api.github.com host.
``pinned_request`` resolves and pins the IP for the life of the request
(closing the DNS-rebinding TOCTOU window) and never follows redirects.
Transport/SSRF failures are transient by nature — they map to
``GitHubError`` so the dispatch retry pipeline handles them. Error text
is static: never the URL (no token in github URLs, unlike Telegram) and
never the response body.
"""
try:
return pinned_request(
method, url, timeout=REQUEST_TIMEOUT, **kwargs)
except (SSRFError, httpx.HTTPError) as e:
raise GitHubError(
f"GitHub request failed ({type(e).__name__})") from None
def _raise_api_error(response, *, not_found_is_create: bool = False) -> None:
"""Map a non-2xx Contents/Repo API response to the typed error ladder.
Error strings are static: they never include the response body (which
can echo the request) and never include the token.
"""
status = response.status_code
if status in (403, 429):
rate_limited = _rate_limit_error(response)
if rate_limited is not None:
raise rate_limited
raise GitHubAuthError(
"GitHub rejected the token or it lacks the"
" \"Contents: Read and write\" permission."
)
if status == 401:
raise GitHubAuthError("GitHub rejected that token.")
if status == 404:
if not_found_is_create:
return # caller: no file yet -> create
raise GitHubNotFoundError(
"GitHub repository or branch not found (or the token cannot see it)."
)
if status in (409, 422):
raise GitHubRejectedError(
f"GitHub refused the write (HTTP {status}); the branch may be"
" protected or the payload invalid."
)
if 300 <= status < 400:
raise GitHubError(
f"GitHub redirected the request (HTTP {status});"
" redirects are not followed"
)
raise GitHubError(f"GitHub request failed (HTTP {status})")
def _headers(token: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
}
def connect(token: str, repo: str, branch: str) -> dict:
"""Validate token + repo + branch with live reads; return account info.
``GET /repos/{repo}`` proves the token and visibility (a ``push: false``
permission is refused up front — connect-time validation is mandatory
because a token that saves but cannot write fails at 6am inside the
scheduler where nobody sees it), then ``GET /repos/{repo}/branches/
{branch}`` proves the branch exists.
Raises ValueError with a user-facing message on refusal, GitHubError on
transient network/5xx failure.
"""
token = normalize_token(token)
repo = parse_repo(repo)
branch = (branch or "").strip()
if not branch:
raise ValueError("Enter the branch name (for example main).")
resp = _request("GET", f"{API_BASE}/repos/{repo}", headers=_headers(token))
if resp.status_code == 401:
raise ValueError("GitHub rejected that token.")
if resp.status_code == 404:
raise ValueError(
"Repository not found, or this token cannot see it."
)
if resp.status_code != 200:
_raise_api_error(resp)
repo_info = resp.json()
perms = repo_info.get("permissions") or {}
if perms.get("push") is False:
raise ValueError(
"This token can read the repository but cannot push to it."
' Give it the "Contents: Read and write" permission.'
)
default_branch = repo_info.get("default_branch") or "main"
full_name = repo_info.get("full_name") or repo
resp = _request(
"GET",
f"{API_BASE}/repos/{repo}/branches/{quote(branch, safe='')}",
headers=_headers(token),
)
if resp.status_code == 404:
raise ValueError(
f"Branch '{branch}' was not found in {full_name}."
f" The repository's default branch is '{default_branch}'."
)
if resp.status_code != 200:
_raise_api_error(resp)
return {
"full_name": full_name,
"private": bool(repo_info.get("private")),
"default_branch": default_branch,
"branch": branch,
"permissions": perms,
}
def read_branch_head(token: str, repo: str, branch: str) -> str:
"""Re-run the branch read; return the head commit sha. Backs the Test
button and internal checks. Raises the typed error ladder."""
resp = _request(
"GET",
f"{API_BASE}/repos/{repo}/branches/{quote(branch, safe='')}",
headers=_headers(token),
)
if resp.status_code != 200:
_raise_api_error(resp)
data = resp.json()
commit = data.get("commit") or {}
return commit.get("sha") or ""
def test_connection(token: str, repo: str, branch: str) -> tuple[bool, str]:
"""The accounts-page Test button: re-read the branch, report the head."""
try:
sha = read_branch_head(token, repo, branch)
except GitHubRateLimitError as e:
return False, str(e)
except GitHubAuthError as e:
return False, str(e)
except GitHubNotFoundError as e:
return False, str(e)
except GitHubRejectedError as e:
return False, str(e)
except GitHubError as e:
return False, str(e)
return True, f"GitHub repository is reachable (branch head {sha[:7]})."
def get_file_sha(token: str, repo: str, branch: str, path: str) -> str | None:
"""GET the file's current blob sha, or None when it does not exist yet.
The GET is idempotent: it may run before the dispatch claim guard.
"""
quoted = _quote_path(path)
resp = _request(
"GET",
f"{API_BASE}/repos/{repo}/contents/{quoted}?ref={quote(branch, safe='')}",
headers=_headers(token),
)
if resp.status_code == 404:
return None
if resp.status_code != 200:
_raise_api_error(resp)
data = resp.json()
if isinstance(data, dict):
return data.get("sha") or None
return None
def put_file(
token: str,
repo: str,
branch: str,
path: str,
content: str,
message: str,
sha: str | None = None,
committer: dict | None = None,
) -> dict:
"""PUT one file (create or update). Returns {sha, html_url, created}.
The size cap is enforced BEFORE any network call: over the cap is a
permanent rejection, and truncating a backup file silently corrupts it.
"""
encoded = _encode_content(content)
body: dict = {
"message": message,
"content": encoded,
"branch": branch,
}
if sha:
body["sha"] = sha
if committer:
body["committer"] = committer
quoted = _quote_path(path)
resp = _request(
"PUT",
f"{API_BASE}/repos/{repo}/contents/{quoted}",
headers=_headers(token),
json=body,
)
if resp.status_code not in (200, 201):
_raise_api_error(resp)
data = resp.json()
commit = data.get("commit") or {}
return {
"sha": commit.get("sha") or data.get("sha") or "",
"html_url": data.get("content", {}).get("html_url")
or data.get("commit", {}).get("html_url")
or "",
"created": resp.status_code == 201,
}
def build_committer(name: str, email: str) -> dict | None:
"""The committer block for the Contents API, or None to commit as the
token's own user. Both fields are required by GitHub, so a half-filled
pair falls back to the token identity rather than a guaranteed 422."""
name = (name or "").strip()
email = (email or "").strip()
if name and email:
return {"name": name, "email": email}
return None
def send_file(
token: str,
repo: str,
branch: str,
path: str,
content: str,
message: str,
committer: dict | None = None,
guard: Callable[[], bool] | None = None,
) -> dict | None:
"""The full two-step Contents API round trip: read the blob sha (create
vs update), then PUT. Returns {sha, html_url, created}.
``guard`` is the dispatch claim check. It runs after the idempotent GET
and immediately before the irreversible PUT; returning False aborts the
write and this returns None. ``_send_github`` passes the real guard, so
the shipped dispatch path is the path these tests exercise.
"""
sha = get_file_sha(token, repo, branch, path)
if guard is not None and not guard():
return None
return put_file(
token, repo, branch, path, content, message,
sha=sha, committer=committer,
)