hm
Rama main · réplica de lectura del repositorio original.
#!/bin/sh
# hm -- the documented steps, in the order that works.
#
# Not a framework and not a new way to run anything: every subcommand below shells the same
# scripts docs/INSTALL.md describes. It exists because the ORDER is load-bearing and getting it
# wrong is silent. Build the ANN index before the rows exist and it is far slower; skip it and
# search still answers, by sequential-scanning every vector for the life of the install. Re-run a
# full ingest over an existing scope and it deletes the documents first, taking every embedding
# with them. Neither raises an error.
#
# ./hm init bring up the docker stack, apply the migrations, print exports
# ./hm migrate [--owner <author>] [--reapply <file>]... [--list | --status]
# apply sql/ in the one order that works, recording each file's
# SHA-256 in hm_schema_migrations. An applied file with the same
# checksum is skipped, so a second run is a no-op. A file that
# CHANGED after it was applied stops the run before anything is
# applied; --reapply <file> says you read the change and want it
# re-run. --owner claims unreferenced legacy entities (default:
# MEM_AUTHOR, else your login). --list prints the order, --status
# what is applied, pending or changed. Connects as HM_ADMIN_URL,
# else DATABASE_URL: it must be the role that owns the tables.
# ./hm author add <author> [<role>]
# create a login the DATABASE maps to <author> (enforced identity:
# it cannot read or write as anyone else, whatever it sets
# mem.reader to) and print how to set its password
# ./hm author remove <role> unmap a login, revoke hm_app from it and disable it
# ./hm author list which logins are mapped to which author (owner only)
# ./hm ingest <dir> <scope> [--walk]
# ingest -> embed -> index, incrementally if the scope already exists.
# --walk enumerates the tree instead of asking git, for a directory
# that is git-ignored or not a repository at all -- without it such a
# directory ingests ZERO documents and says why.
# A `.hmignore` in <dir> keeps paths OUT of the corpus (one
# pattern per line, trailing '/' = whole subtree). For the file of
# credentials and the superseded spec set that live next to the real
# documents: --walk cannot exclude them, it is all-or-nothing.
# ./hm refresh re-ingest every scope in $HM_SCOPES (default
# ~/.config/hypermnesia/scopes: <dir><TAB><scope><TAB>[--walk] per
# line), then assert no chunk was left without an embedding. This is
# the one to put on a timer -- freshness.py reports drift, this ends
# it.
# ./hm install [--wire] [--ref <commit>] [--console[=<features>]] [--dry-run] ...
# install a COMMIT of this checkout as the machine's release:
# ~/.HyperMnesia/current (HYPERMNESIA_HOME moves it), with the MCP
# server built into it. Hooks, jobs and MCP configs then run that,
# not this working copy. --wire rewrites the configs that still point
# into a checkout (each one backed up first). ci/install.py --help.
# ./hm doctor is this install actually working, or only appearing to
# ./hm latency where the time goes: hook, embedder, database, reranker
#
# Env: DATABASE_URL (required by ingest/doctor; init prints the one it just set up).
# HM_ADMIN_URL (migrate, author: the table owner's connection; default DATABASE_URL).
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
PY=${HM_PYTHON:-python3}
die() { printf '%s\n' "hm: $*" >&2; exit 1; }
step() { printf '\n== %s\n' "$*"; }
need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not on PATH"; }
# The header comment above IS the help text; printing it from the file keeps the two from
# drifting. Stop at the first line that is not a comment rather than a hardcoded line number,
# which goes stale the moment anyone edits the header.
usage() {
awk 'NR > 1 && /^#/ { sub(/^# ?/, ""); print; next } NR > 1 { exit }' "$0"
exit "${1:-1}"
}
# -- migrations -------------------------------------------------------------------------------
# THE apply order, and the only copy of it. CI, the docs and ci/doctor.py read it from here
# (`./hm migrate --list`) instead of keeping their own: there used to be four copies, and nothing
# noticed when one fell behind.
#
# schema.sql documents, chunks, the component map
# migrate_schema_version.sql hm_schema_migrations: what has been applied, with checksums
# schema_mem.sql .. dreamer personal memory, then its migrations in dependency order
# migrate_author_roles.sql per-author logins (enforced identity)
# mem_app_role.sql LAST: grants on, and re-states the policies for, all of it
MIGRATIONS="schema.sql
migrate_schema_version.sql
schema_mem.sql
mem_multiuser.sql
migrate_entity_ownership.sql
migrate_memory_perspectives.sql
migrate_memory_evidence.sql
migrate_peer_cards.sql
migrate_dreamer.sql
migrate_memory_lifecycle.sql
migrate_constraint_sources.sql
migrate_author_roles.sql
mem_app_role.sql"
# Re-applied whenever anything before it was applied in the same run, and whenever it changed:
# it grants on everything above it, so a table added without re-running it is unreachable to the
# application -- which looks like an empty store, not like an error.
REPEATABLE="mem_app_role.sql"
# Tolerated when absent, with a note. Every other missing file is an error.
MAY_BE_ABSENT=""
SQL_DIR=${HM_SQL_DIR:-$ROOT/sql}
# How migrate reaches the database: `psql "$MIGRATE_URL"` from this host by default; `hm init`
# switches it to the psql inside the compose container, which needs nothing installed here.
MIGRATE_VIA=url
MIGRATE_URL=
mpsql() {
if [ "$MIGRATE_VIA" = compose ]; then
docker compose -f "$ROOT/deploy/docker/docker-compose.yml" exec -T postgres \
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" "$@"
else
psql "$MIGRATE_URL" "$@"
fi
}
in_list() { # in_list <word> <space-separated list>
case " $2 " in *" $1 "*) return 0 ;; esac
return 1
}
sha256_of() {
if command -v sha256sum >/dev/null 2>&1; then s=$(sha256sum < "$1")
elif command -v shasum >/dev/null 2>&1; then s=$(shasum -a 256 < "$1")
else s=$(openssl dgst -sha256 -r < "$1"); fi
s=${s%% *}
case "$s" in ''|*[!0-9a-f]*) die "could not checksum $1" ;; esac
[ "${#s}" -eq 64 ] || die "could not checksum $1"
printf %s "$s"
}
recorded_sum() { # the checksum recorded for <file>, or nothing
awk -v n="$1" '$1 == n { print $2 }' "$mtmp/recorded"
}
record_applied() { # record_applied <file> <sha256>
# Both values are safe to interpolate: the name comes from MIGRATIONS, the sum is 64 hex.
mpsql -tAX -q -v ON_ERROR_STOP=1 -c "INSERT INTO public.hm_schema_migrations (name, checksum)
VALUES ('$1', '$2') ON CONFLICT (name)
DO UPDATE SET checksum = EXCLUDED.checksum, applied_at = now()" >/dev/null \
|| die "sql/$1 was applied but could not be recorded -- re-run ./hm migrate"
}
cmd_migrate() {
owner=${MEM_AUTHOR:-$(id -un)}
reapply=""
mode=apply
while [ $# -gt 0 ]; do
case "$1" in
--list) mode=list ;;
--status) mode=status ;;
--owner) [ $# -ge 2 ] || usage; owner=$2; shift ;;
--owner=*) owner=${1#--owner=} ;;
--reapply) [ $# -ge 2 ] || usage; reapply="$reapply ${2##*/}"; shift ;;
--reapply=*) v=${1#--reapply=}; reapply="$reapply ${v##*/}" ;;
*) usage ;;
esac
shift
done
if [ "$mode" = list ]; then
printf '%s\n' $MIGRATIONS
return 0
fi
all=$(printf '%s ' $MIGRATIONS)
for r in $reapply; do
in_list "$r" "$all" || die "--reapply $r: not a migration (see ./hm migrate --list)"
done
# The owner value lands in a psql variable that one migration interpolates as a literal.
case "$owner" in
*[!A-Za-z0-9._-]*|"") die "--owner must match [A-Za-z0-9._-]+ (got: $owner)" ;;
esac
if [ "$MIGRATE_VIA" = url ]; then
need psql
MIGRATE_URL=${HM_ADMIN_URL:-${DATABASE_URL:-}}
[ -n "$MIGRATE_URL" ] || die "set HM_ADMIN_URL (or DATABASE_URL) to the table owner's connection string"
fi
mtmp=$(mktemp -d); trap 'rm -rf "$mtmp"' EXIT
if ! has=$(mpsql -tAX -v ON_ERROR_STOP=1 \
-c "SELECT to_regclass('public.hm_schema_migrations') IS NOT NULL" 2>&1); then
die "could not reach the database: $has"
fi
: > "$mtmp/recorded"
if [ "$has" = t ]; then
mpsql -tAX -v ON_ERROR_STOP=1 -F ' ' \
-c "SELECT name, checksum FROM public.hm_schema_migrations ORDER BY name" \
> "$mtmp/recorded" || die "could not read hm_schema_migrations"
fi
# Pass 1: decide everything before applying anything. A changed file found halfway through
# would leave this run half applied and half refused.
changed=""
for f in $MIGRATIONS; do
path="$SQL_DIR/$f"
if [ ! -f "$path" ]; then
in_list "$f" "$MAY_BE_ABSENT" || die "sql/$f is missing -- this checkout is incomplete"
[ "$mode" != status ] || printf '%-32s absent from this checkout\n' "$f"
continue
fi
sum=$(sha256_of "$path")
rec=$(recorded_sum "$f")
state=pending
if [ -n "$rec" ] && [ "$rec" = "$sum" ]; then
state=applied
elif [ -n "$rec" ]; then
if in_list "$f" "$REPEATABLE"; then
state="changed (re-applied by the next run)"
elif in_list "$f" "$reapply"; then
state="changed (--reapply)"
else
state=CHANGED
changed="$changed $f"
fi
fi
[ "$mode" != status ] || printf '%-32s %s\n' "$f" "$state"
done
[ "$mode" != status ] || return 0
if [ -n "$changed" ]; then
printf '!! changed since they were applied:%s\n' "$changed" >&2
printf '!! An applied migration is not replayed silently. Read the change, then run\n' >&2
printf '!! ./hm migrate --reapply <file>\n' >&2
printf '!! for each one (every file in sql/ is written to be re-runnable).\n' >&2
die "refusing to migrate: nothing was applied"
fi
# Pass 2: apply. hm_schema_migrations does not exist until migrate_schema_version.sql has
# run, so the files applied before it are recorded as soon as it has.
recordable=$has
owed=""
unrecorded=""
applied=0
current=0
for f in $MIGRATIONS; do
path="$SQL_DIR/$f"
if [ ! -f "$path" ]; then
printf -- '-- sql/%s is not in this checkout -- skipped\n' "$f"
continue
fi
sum=$(sha256_of "$path")
rec=$(recorded_sum "$f")
# Current, unless it was named in --reapply, or it is the grant file and something
# before it has just been applied.
if [ "$rec" = "$sum" ] && ! in_list "$f" "$reapply" \
&& { ! in_list "$f" "$REPEATABLE" || [ "$applied" -eq 0 ]; }; then
current=$((current + 1))
continue
fi
# Anything applied before the grant file can (re)install a policy or a table it has not
# restated yet, and until it runs again the store is not in the shape it certifies.
# Forget its record FIRST: a run interrupted between this file and the grant file then
# still owes it, instead of finding every checksum current and applying nothing.
if [ "$recordable" = t ] && ! in_list "$f" "$REPEATABLE" && [ -z "$owed" ]; then
for r in $REPEATABLE; do
mpsql -tAX -q -v ON_ERROR_STOP=1 \
-c "DELETE FROM public.hm_schema_migrations WHERE name = '$r'" >/dev/null \
|| die "could not mark sql/$r as owed -- nothing was applied"
done
owed=t
fi
printf -- '-- applying sql/%s\n' "$f"
# From stdin, so the same call works against the compose container, which sees the
# files under another path. The output is shown only on failure: an idempotent re-run
# prints a page of "already exists, skipping" notices that hide the line that matters.
if ! mpsql -X -q -v ON_ERROR_STOP=1 -v owner="$owner" < "$path" > "$mtmp/out" 2>&1; then
cat "$mtmp/out" >&2
die "sql/$f failed -- nothing after it was applied"
fi
applied=$((applied + 1))
[ "$f" != migrate_schema_version.sql ] || recordable=t
unrecorded="$unrecorded $f:$sum"
if [ "$recordable" = t ]; then
for u in $unrecorded; do
record_applied "${u%%:*}" "${u#*:}"
done
unrecorded=""
fi
done
[ -z "$unrecorded" ] || die "applied but not recorded (no hm_schema_migrations):$unrecorded"
if [ "$applied" -eq 0 ]; then
printf -- '-- nothing to apply: all %s migration(s) are recorded with their current checksum\n' "$current"
else
printf '\nmigrations: %s applied, %s already current\n' "$applied" "$current"
fi
}
# -- per-author logins ------------------------------------------------------------------------
cmd_author() {
[ $# -ge 1 ] || usage
sub=$1; shift
# Arguments first, the database second: a bad name is refused before anything connects.
case "$sub" in
add)
[ $# -ge 1 ] && [ $# -le 2 ] || usage
case "$1" in *[!A-Za-z0-9._-]*|"") die "author must match [A-Za-z0-9._-]+ (got: $1)" ;; esac
case "${2:-}" in *[!a-z0-9_]*) die "role must match [a-z0-9_]+ (got: $2)" ;; esac ;;
remove)
[ $# -eq 1 ] || usage
case "$1" in *[!a-z0-9_]*|"") die "role must match [a-z0-9_]+ (got: $1)" ;; esac ;;
list) ;;
*) usage ;;
esac
need psql
url=${HM_ADMIN_URL:-${DATABASE_URL:-}}
[ -n "$url" ] || die "set HM_ADMIN_URL (or DATABASE_URL) to the table owner's connection string"
atmp=$(mktemp -d); trap 'rm -rf "$atmp"' EXIT
case "$sub" in
add)
name=$1; role=${2:-}
if ! r=$(printf "SELECT mem.add_author_role(:'a', NULLIF(:'r', '')::name);\n" \
| psql "$url" -X -tA -q -v ON_ERROR_STOP=1 -v a="$name" -v r="$role" \
2> "$atmp/err"); then
die "could not create a login for $name: $(cat "$atmp/err")"
fi
cat <<MSG
Login '$r' now reads and writes as '$name'. The database fixes that identity: the mem.reader
setting is ignored for this login, so it cannot read or write as anyone else.
Set its password (prompted; nothing lands in a file or in shell history):
psql "\${HM_ADMIN_URL:-\$DATABASE_URL}" -c '\\password $r'
Then, on $name's machine and in every job that runs as $name:
DATABASE_URL=postgresql://$r:<password>@<host>:<port>/<database>
MEM_AUTHOR=$name # must match: a write under any other name is refused
and check what the database says from there:
psql "\$DATABASE_URL" -tAXc 'SELECT * FROM mem.whoami()' # $r|$name|enforced
Project membership is separate:
INSERT INTO mem.project_members (author, project) VALUES ('$name', '<tag>');
The boundary holds only while nobody else uses the shared hm_app password (docs/DESIGN.md).
MSG
;;
remove)
if ! r=$(printf "SELECT mem.remove_author_role(:'r');\n" \
| psql "$url" -X -tA -q -v ON_ERROR_STOP=1 -v r="$1" 2> "$atmp/err"); then
die "could not remove $1: $(cat "$atmp/err")"
fi
[ "$r" = t ] || die "$1 is not a mapped author login"
printf 'Unmapped %s, revoked hm_app from it and disabled its login.\n' "$1"
;;
list)
psql "$url" -X -v ON_ERROR_STOP=1 \
-c "SELECT role, author, created_at FROM mem.author_roles ORDER BY author, role"
;;
esac
}
cmd_init() {
need docker
env_file="$ROOT/deploy/docker/.env"
if [ ! -f "$env_file" ]; then
step "Writing $env_file"
cp "$ROOT/deploy/docker/.env.example" "$env_file"
# Generated, not chosen: a password typed in a hurry outlives the hurry.
# Hex, not base64: a '/' or '+' in the password has to be percent-encoded in the
# postgresql:// URI this prints, and a password that silently breaks the URI is worse
# than a shorter one.
pw=$(openssl rand -hex 16 2>/dev/null | tr -d '\n') \
|| die "could not generate a password -- set POSTGRES_PASSWORD in $env_file yourself"
[ -n "$pw" ] || die "empty password generated -- set POSTGRES_PASSWORD in $env_file yourself"
# BSD and GNU sed disagree about -i, so write a new file and move it.
sed "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=$pw|" "$env_file" > "$env_file.tmp"
mv "$env_file.tmp" "$env_file"
chmod 600 "$env_file"
else
step "Keeping the existing $env_file" # never clobber a password already in use
fi
# shellcheck disable=SC1090
. "$env_file"
: "${POSTGRES_USER:=hm}" "${POSTGRES_DB:=hypermnesia}" "${PG_PORT:=5432}"
[ -n "${POSTGRES_PASSWORD:-}" ] || die "POSTGRES_PASSWORD is empty in $env_file"
step "Starting Postgres and the embedder"
(cd "$ROOT/deploy/docker" && docker compose up -d)
step "Waiting for Postgres to accept connections"
i=0
until docker compose -f "$ROOT/deploy/docker/docker-compose.yml" \
exec -T postgres pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; do
i=$((i + 1))
[ "$i" -lt 60 ] || die "Postgres did not come up in 60s -- 'docker compose logs postgres'"
sleep 1
done
# Every file is part of the current memory contract. Loading only the base schema leaves the
# MCP catalog advertising perspectives, evidence and grounded context against tables that do
# not exist. The owner value is used only to claim unreferenced rows from an older store.
step "Applying the migrations (./hm migrate, through the container's psql)"
MIGRATE_VIA=compose
cmd_migrate --owner "${MEM_AUTHOR:-$(id -un)}"
url="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@localhost:$PG_PORT/$POSTGRES_DB"
cat <<EOF
Up. Export these, in this shell and in your MCP client's env:
export DATABASE_URL='$url'
export EMBED_BACKEND=tei TEI_URL=http://localhost:${TEI_PORT:-8080}
Then put a repo in: ./hm ingest ~/code/myrepo myrepo
EOF
}
cmd_ingest() {
# --walk anywhere among the arguments: a notes folder that is git-ignored is the common case
# for it, and that is exactly the case which otherwise ingests nothing at all.
#
# Read with `case` rather than a `[ ]` test, and the positionals rebuilt by assignment rather
# than by `set -- $(...)`: under `set -e` a failing test as the last statement of a loop ends
# the script, and an unquoted expansion loses a directory whose name contains a space.
walk=""; dir=""; scope=""; n=0
for a in "$@"; do
case "$a" in
--walk) walk="--walk" ;;
*) n=$((n + 1))
case "$n" in
1) dir="$a" ;;
2) scope="$a" ;;
esac ;;
esac
done
[ "$n" -eq 2 ] || usage
[ -d "$dir" ] || die "no such directory: $dir"
# Before the first psql call, not inside the python step that runs after it: the scope is
# interpolated into a single-quoted SQL literal below, so a quote in it produces a syntax
# error at best and executes as written at worst. ingest_repo.py checks the same alphabet,
# but by then two queries have already run.
case "$scope" in
*[!A-Za-z0-9._-]*|"") die "scope must match [A-Za-z0-9._-]+ (got: $scope)" ;;
esac
need psql
[ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"
tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
# Incremental when the scope already holds documents. The full path OPENS with
# DELETE FROM documents WHERE repo=<scope>, which cascades to chunks and therefore to every
# embedding -- a one-file edit costing a full re-embed, quietly. Taking the snapshot from the
# database we are about to write to is the point: one from anywhere else would mark documents
# "already stored" that were never there.
# Do not collapse this into `|| die "cannot reach the store"`: a query that fails for any
# other reason would then send the reader to debug Postgres connectivity while the store is
# perfectly healthy.
if ! n=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
-c "SELECT count(*) FROM documents WHERE repo='$scope'" 2>&1); then
die "could not count documents for scope '$scope': $n"
fi
if [ "$n" -gt 0 ]; then
step "Scope '$scope' already holds $n document(s) -- ingesting incrementally"
psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 -F' ' \
-c "SELECT path, content_hash FROM documents WHERE repo='$scope'" > "$tmp/known.tsv"
"$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" \
--known-hashes "$tmp/known.tsv" $walk
else
step "First ingest of scope '$scope'"
"$PY" "$ROOT/ingest/ingest_repo.py" "$dir" "$scope" "$tmp/out.sql" $walk
fi
step "Loading the documents"
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q -f "$tmp/out.sql"
step "Embedding what has no vector yet"
# Not fatal here, and not ignored either. The documents are in and the index should still be
# built, so the run carries on -- but a batch the embedder refused leaves chunks that only the
# lexical leg can ever find, and the run must end non-zero saying so.
embed_rc=0
EMBED_REPO="$scope" "$PY" "$ROOT/ingest/embed_chunks.py" || embed_rc=$?
# After the rows exist, never before: building HNSW on an empty table then filling it is far
# slower than the other way round. IF NOT EXISTS makes the re-run a no-op.
step "Building the ANN index if it is missing"
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -q \
-c "CREATE INDEX IF NOT EXISTS chunks_embedding_hnsw ON chunks USING hnsw (embedding vector_cosine_ops)"
step "Checking the result"
HM_REPO="$scope" "$PY" "$ROOT/ci/doctor.py" || true
[ "$embed_rc" -eq 0 ] || die "embedding failed: some chunks of '$scope' were not embedded (embed_chunks exit $embed_rc); they are found by exact words only until './hm ingest' is re-run with the embedder up"
cat <<EOF
Documents are in. What is NOT in is the part that pays: the Tier 0/1 map -- which files belong to
which component, and which rules must hold for them. Nothing can generate that honestly from a
directory listing. See examples/seed_example.sql and skills/onboard-project.
EOF
}
cmd_refresh() {
# Every scope, in one pass, from a list -- because the thing that rots a store is not a bad
# ingest, it is a good one that nobody re-runs. `ci/freshness.py` already counts the drift and
# writes it down; counting is not fixing, and a number in a log fixes nothing by itself.
#
# The list is tab-separated, one scope per line, `#` comments allowed:
#
# /home/me/notes<TAB>notes<TAB>--walk
# /home/me/src/api<TAB>api
#
# Tabs, not spaces: a directory with a space in its name is ordinary, and splitting on
# whitespace would quietly ingest the wrong path or nothing at all.
file=${HM_SCOPES:-$HOME/.config/hypermnesia/scopes}
[ -f "$file" ] || die "no scope list at $file -- one '<dir><TAB><scope><TAB>[--walk]' per line (or set HM_SCOPES)"
[ -n "${DATABASE_URL:-}" ] || die "DATABASE_URL is not set (./hm init prints it)"
need psql
# One refresh at a time. `mkdir` is the atomic test-and-set every shell has. Two overlapping
# runs would ingest the same scopes concurrently, and the second would delete and rebuild
# chunks the first is embedding.
lock="${TMPDIR:-/tmp}/hm-refresh.lock"
if ! mkdir "$lock" 2>/dev/null; then
die "another refresh holds $lock"
fi
trap 'rmdir "$lock" 2>/dev/null' EXIT INT TERM
failed=""
listed=""
while IFS= read -r line; do
case "$line" in ''|\#*) continue ;; esac
dir=$(printf %s "$line" | cut -f1)
scope=$(printf %s "$line" | cut -f2)
extra=$(printf %s "$line" | cut -f3)
[ -n "$scope" ] || die "malformed line in $file (need a TAB between the path and the scope): $line"
# A scope on the list and not on disk is a misconfiguration, not a reason to skip
# quietly: the list is hand-written, so a renamed or unmounted directory would drop a
# whole scope out of every refresh from then on while the run still reported success.
if [ ! -d "$dir" ]; then
printf '!! %s: no such directory -- scope %s was NOT refreshed\n' "$dir" "$scope" >&2
failed="$failed $scope:missing"
continue
fi
step "Refreshing $scope from $dir"
# In a subshell, because cmd_ingest calls die() on a bad scope or an unreachable store and
# one bad entry must not take the rest of the list with it.
#
# `set +e` around it, and `set -e` INSIDE it, and neither is decoration. A subshell that
# is the condition of `if` or the left side of `||` has errexit suppressed for everything
# in it -- and re-setting it inside does not bring it back (checked in dash and in bash's
# sh mode). Written as `if ! ( cmd_ingest ... )`, the embed step could die with a
# traceback and the ingest would carry on to build the index and report success. It did:
# the first run of this command against a live store crashed on a missing psycopg2, said
# nothing, and was only caught by the unembedded-chunk assertion at the very end.
set +e
# shellcheck disable=SC2086 # $extra is one optional flag or nothing, by construction
( set -e; cmd_ingest "$dir" "$scope" $extra )
rc=$?
set -e
[ "$rc" -eq 0 ] || failed="$failed $scope"
listed="${listed:+$listed,}$scope"
done < "$file"
# Which scopes does the STORE hold that this list never names? Nothing ties the two together:
# a directory ingested once by hand and never added to the list is never refreshed again, and
# search keeps answering about it from whatever it held that day -- with no error, no empty
# result and no sign that anything is wrong. That is this project's whole subject matter, and
# the refresh command had the same blind spot it was written to close.
#
# A note and not a failure: a scope whose directory is gone for good is a legitimate resident
# of the store, its documents still searchable history. Failing the run nightly over one would
# teach whoever reads the output to stop reading it.
step "Checking the list against the store"
for r in $(psql "$DATABASE_URL" -tAX -c \
"SELECT repo FROM documents GROUP BY repo ORDER BY repo" 2>/dev/null); do
case ",$listed," in *",$r,"*) continue ;; esac
printf -- '-- %s: in the store, not in %s -- nothing refreshes it\n' "$r" "$file"
done
# The check that makes the loop mean something. A chunk with no embedding raises no error
# anywhere: it is simply absent from the vector half of every search, while the document it
# belongs to still counts as indexed and the scope still counts as refreshed.
step "Checking that nothing is left unembedded"
if ! left=$(psql "$DATABASE_URL" -tAX -v ON_ERROR_STOP=1 \
-c "SELECT count(*) FROM chunks WHERE embedding IS NULL" 2>&1); then
printf '!! could not count unembedded chunks: %s\n' "$left" >&2
failed="$failed count"
else
case "$left" in
0) printf -- '-- all chunks embedded\n' ;;
''|*[!0-9]*) printf '!! unexpected count output: %s\n' "$left" >&2; failed="$failed count" ;;
*) printf '!! %s chunk(s) left without an embedding\n' "$left" >&2
failed="$failed embeddings:$left" ;;
esac
fi
[ -z "$failed" ] || die "refresh finished with problems:$failed"
printf '\nrefresh clean\n'
}
[ $# -ge 1 ] || usage 0
sub=$1; shift
case "$sub" in
init) cmd_init "$@" ;;
ingest) cmd_ingest "$@" ;;
refresh) cmd_refresh "$@" ;;
migrate) cmd_migrate "$@" ;;
author) cmd_author "$@" ;;
install) exec "$PY" "$ROOT/ci/install.py" "$@" ;;
doctor) exec "$PY" "$ROOT/ci/doctor.py" "$@" ;;
latency) exec "$PY" "$ROOT/ci/latency.py" "$@" ;;
-h|--help|help) usage 0 ;;
*) die "unknown command: $sub (try ./hm help)" ;;
esac