chore(dev): one-shot dev setup script + fix control-plane production build (#1910)
* fix(control-plane): force NODE_ENV=production for production build A globally-exported NODE_ENV=development (common in dev shells) overrides Next.js's production default during `next build`, bundling React's development build under the production server renderer. Static prerendering then crashes with "Cannot read properties of null (reading 'useContext')" — even on the built-in _global-error page. Pin NODE_ENV=production for the build step so it is robust regardless of the caller's shell. Docker is unaffected (it invokes next build directly in a clean env). * chore(dev): add one-shot dev environment setup script Add scripts/dev/setup.sh: an idempotent bootstrap that installs the required toolchains (uv/Python, Node/npm, Rust/cargo) when missing, creates .env, configures git hooks, installs all Python + Node workspace deps, pre-downloads the local ML models + tokenizer for offline use, and builds the TypeScript SDK and Rust CLI. Flags: --skip-build, --skip-models, --with-docs, --force. Document it in CONTRIBUTING.md as the recommended setup, keeping the manual steps as a fallback.
This commit is contained in:
Nicolò Boschi
+25
-2
@@ -9,13 +9,36 @@ Thanks for your interest in contributing to Hindsight!
|
||||
git clone [email protected]:vectorize-io/hindsight.git
|
||||
cd hindsight
|
||||
```
|
||||
2. Set up your environment:
|
||||
|
||||
2. Bootstrap your dev environment in one shot:
|
||||
```bash
|
||||
./scripts/dev/setup.sh
|
||||
```
|
||||
This is idempotent (safe to re-run) and gets you ready to develop, including
|
||||
offline. It:
|
||||
- installs the required toolchains if missing (uv/Python, Node/npm, Rust/cargo),
|
||||
- creates `.env` from `.env.example` (remember to add your LLM API key),
|
||||
- configures git hooks,
|
||||
- installs all Python and Node workspace dependencies,
|
||||
- pre-downloads the local ML models + tokenizer so the API runs offline,
|
||||
- builds the TypeScript SDK and the Rust CLI.
|
||||
|
||||
Useful flags: `--skip-build` (deps only), `--skip-models` (skip ML model
|
||||
download), `--with-docs` (also build the docs site), `--force` (rebuild
|
||||
artifacts). Docker image builds are out of scope. Run
|
||||
`./scripts/dev/setup.sh --help` for details.
|
||||
|
||||
### Manual setup
|
||||
|
||||
If you'd rather set things up by hand instead of running the script above:
|
||||
|
||||
1. Set up your environment:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
Edit the .env to add LLM API key and config as required
|
||||
|
||||
3. Install dependencies:
|
||||
2. Install dependencies:
|
||||
```bash
|
||||
# Python dependencies
|
||||
uv sync --directory hindsight-api/
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "next dev --turbopack -p ${PORT:-9999}",
|
||||
"build": "next build && npm run build:standalone",
|
||||
"build": "NODE_ENV=production next build && npm run build:standalone",
|
||||
"build:standalone": "rm -rf standalone && SERVER_JS=$(find .next/standalone -path '*/node_modules' -prune -o -name 'server.js' -print | head -1) && test -n \"$SERVER_JS\" || (echo 'Error: server.js not found in .next/standalone - standalone build failed' && exit 1) && STANDALONE_ROOT=$(dirname \"$SERVER_JS\") && cp -r \"$STANDALONE_ROOT\" standalone && cp -r .next/standalone/node_modules standalone/node_modules && mkdir -p standalone/.next && cp -r .next/static standalone/.next/static && mkdir -p standalone/public && (cp -r public/* standalone/public/ 2>/dev/null || true)",
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
|
||||
Executable
+341
@@ -0,0 +1,341 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# One-shot dev environment setup for Hindsight.
|
||||
#
|
||||
# Installs every toolchain the repo needs (uv/Python, Node/npm, Rust/cargo),
|
||||
# installs all workspace dependencies, and builds the core artifacts so the
|
||||
# project is ready to develop with — including offline, after this finishes.
|
||||
#
|
||||
# Each step checks whether it's already done and skips it, so the script is
|
||||
# safe to re-run. Docker image builds are intentionally out of scope.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/dev/setup.sh # install toolchains + deps + core builds
|
||||
# ./scripts/dev/setup.sh --skip-build # install toolchains + deps only
|
||||
# ./scripts/dev/setup.sh --skip-models # don't pre-download local ML models
|
||||
# ./scripts/dev/setup.sh --with-docs # also build the docs site
|
||||
# ./scripts/dev/setup.sh --force # rebuild artifacts even if present
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# --- options ----------------------------------------------------------------
|
||||
SKIP_BUILD=false
|
||||
SKIP_MODELS=false
|
||||
WITH_DOCS=false
|
||||
FORCE=false
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--skip-build) SKIP_BUILD=true ;;
|
||||
--skip-models) SKIP_MODELS=true ;;
|
||||
--with-docs) WITH_DOCS=true ;;
|
||||
--force) FORCE=true ;;
|
||||
-h|--help)
|
||||
sed -n '3,17p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Unknown option: $arg (try --help)"; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --- pretty logging ---------------------------------------------------------
|
||||
step() { printf '\n\033[1;34m▶ %s\033[0m\n' "$1"; }
|
||||
ok() { printf ' \033[32m✓\033[0m %s\n' "$1"; }
|
||||
info() { printf ' \033[2m• %s\033[0m\n' "$1"; }
|
||||
warn() { printf ' \033[33m⚠ %s\033[0m\n' "$1"; }
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
OS="$(uname -s)"
|
||||
SUMMARY=()
|
||||
|
||||
# Minimum Node major version the workspaces expect (CI builds on 20/22).
|
||||
NODE_MIN_MAJOR=20
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Toolchains
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ensure_uv() {
|
||||
step "uv (Python toolchain & package manager)"
|
||||
# uv installs to ~/.local/bin by default — make sure it's reachable now.
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
if have uv; then
|
||||
ok "already installed ($(uv --version))"
|
||||
SUMMARY+=("uv: present")
|
||||
return
|
||||
fi
|
||||
info "installing via astral.sh installer..."
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
# The installer writes an env file with the PATH update; source it so the
|
||||
# rest of this run can use uv immediately.
|
||||
[ -f "$HOME/.local/bin/env" ] && . "$HOME/.local/bin/env"
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
have uv || { warn "uv not on PATH after install — open a new shell and re-run"; exit 1; }
|
||||
ok "installed ($(uv --version))"
|
||||
SUMMARY+=("uv: installed")
|
||||
}
|
||||
|
||||
node_major() { node -p 'process.versions.node.split(".")[0]' 2>/dev/null || echo 0; }
|
||||
|
||||
ensure_node() {
|
||||
step "Node.js + npm"
|
||||
if have node && [ "$(node_major)" -ge "$NODE_MIN_MAJOR" ]; then
|
||||
ok "already installed (node $(node --version), npm $(npm --version))"
|
||||
SUMMARY+=("node: present ($(node --version))")
|
||||
return
|
||||
fi
|
||||
if have node; then
|
||||
warn "node $(node --version) is older than v${NODE_MIN_MAJOR} — upgrading"
|
||||
fi
|
||||
|
||||
# Pick the most appropriate installer for the host. Order matters: prefer a
|
||||
# system package manager, fall back to nvm (no root, works in containers).
|
||||
if [ "$OS" = "Darwin" ] && have brew; then
|
||||
info "installing via Homebrew..."
|
||||
brew install node
|
||||
elif have apt-get; then
|
||||
info "installing via apt (NodeSource ${NODE_MIN_MAJOR}.x)..."
|
||||
local SUDO=""
|
||||
[ "$(id -u)" -ne 0 ] && have sudo && SUDO="sudo"
|
||||
curl -fsSL "https://deb.nodesource.com/setup_${NODE_MIN_MAJOR}.x" | $SUDO bash -
|
||||
$SUDO apt-get install -y nodejs
|
||||
else
|
||||
info "installing via nvm..."
|
||||
export NVM_DIR="$HOME/.nvm"
|
||||
if [ ! -s "$NVM_DIR/nvm.sh" ]; then
|
||||
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
|
||||
fi
|
||||
# shellcheck disable=SC1091
|
||||
. "$NVM_DIR/nvm.sh"
|
||||
nvm install "$NODE_MIN_MAJOR"
|
||||
nvm alias default "$NODE_MIN_MAJOR"
|
||||
fi
|
||||
|
||||
have node || { warn "node not on PATH after install — open a new shell and re-run"; exit 1; }
|
||||
ok "installed (node $(node --version), npm $(npm --version))"
|
||||
SUMMARY+=("node: installed ($(node --version))")
|
||||
}
|
||||
|
||||
ensure_rust() {
|
||||
step "Rust + cargo (for hindsight-cli)"
|
||||
[ -f "$HOME/.cargo/env" ] && . "$HOME/.cargo/env"
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
if have cargo; then
|
||||
ok "already installed ($(cargo --version))"
|
||||
SUMMARY+=("cargo: present")
|
||||
return
|
||||
fi
|
||||
info "installing via rustup..."
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path
|
||||
. "$HOME/.cargo/env"
|
||||
have cargo || { warn "cargo not on PATH after install — open a new shell and re-run"; exit 1; }
|
||||
ok "installed ($(cargo --version))"
|
||||
SUMMARY+=("cargo: installed")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Repo bootstrap
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ensure_env_file() {
|
||||
step ".env file"
|
||||
if [ -f "$ROOT_DIR/.env" ]; then
|
||||
ok ".env already exists"
|
||||
return
|
||||
fi
|
||||
cp "$ROOT_DIR/.env.example" "$ROOT_DIR/.env"
|
||||
ok "created .env from .env.example"
|
||||
warn "set HINDSIGHT_API_LLM_API_KEY (and provider/model) in .env before running the API"
|
||||
SUMMARY+=(".env: created — add your LLM API key")
|
||||
}
|
||||
|
||||
setup_git_hooks() {
|
||||
step "Git hooks"
|
||||
if [ "$(git config --get core.hooksPath || true)" = "$ROOT_DIR/.githooks" ]; then
|
||||
ok "hooks already configured"
|
||||
return
|
||||
fi
|
||||
./scripts/setup-hooks.sh >/dev/null
|
||||
ok "configured core.hooksPath -> .githooks"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dependencies (also primes offline caches)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
install_python_deps() {
|
||||
step "Python dependencies (uv sync — whole workspace)"
|
||||
# Installs the pinned interpreter (.python-version) if needed plus every
|
||||
# workspace member's deps into .venv. Populates uv's cache for offline use.
|
||||
uv sync
|
||||
ok "Python workspace synced (.venv ready)"
|
||||
SUMMARY+=("python deps: synced")
|
||||
}
|
||||
|
||||
install_node_deps() {
|
||||
step "Node dependencies (npm ci — all workspaces)"
|
||||
if [ -d "$ROOT_DIR/node_modules" ] && [ "$FORCE" = false ]; then
|
||||
# npm ci is the clean-install path; only skip the reinstall when the
|
||||
# tree already exists and the user didn't ask to force.
|
||||
info "node_modules present — running npm install to reconcile lockfile"
|
||||
npm install --no-audit --no-fund
|
||||
else
|
||||
npm ci --no-audit --no-fund
|
||||
fi
|
||||
ok "Node workspaces installed"
|
||||
SUMMARY+=("node deps: installed")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ML model cache (offline readiness) — mirrors docker/standalone/Dockerfile
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
prewarm_models() {
|
||||
step "Pre-download ML models & tokenizer (offline readiness)"
|
||||
# Respect provider/model overrides from .env; default to local + repo
|
||||
# defaults (BAAI/bge-small-en-v1.5, cross-encoder/ms-marco-MiniLM-L-6-v2).
|
||||
set -a; [ -f "$ROOT_DIR/.env" ] && . "$ROOT_DIR/.env"; set +a
|
||||
|
||||
# Only the "local" provider downloads weights; remote providers (TEI,
|
||||
# OpenAI, Cohere, ...) fetch nothing here. tiktoken is always cached
|
||||
# because token counting runs regardless of provider.
|
||||
local emb_model="" rer_model=""
|
||||
if [ "${HINDSIGHT_API_EMBEDDINGS_PROVIDER:-local}" = "local" ]; then
|
||||
emb_model="${HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL:-BAAI/bge-small-en-v1.5}"
|
||||
info "embeddings (local): $emb_model"
|
||||
else
|
||||
info "embeddings provider is '${HINDSIGHT_API_EMBEDDINGS_PROVIDER}' — no local weights to fetch"
|
||||
fi
|
||||
if [ "${HINDSIGHT_API_RERANKER_PROVIDER:-local}" = "local" ]; then
|
||||
rer_model="${HINDSIGHT_API_RERANKER_LOCAL_MODEL:-cross-encoder/ms-marco-MiniLM-L-6-v2}"
|
||||
info "reranker (local): $rer_model"
|
||||
else
|
||||
info "reranker provider is '${HINDSIGHT_API_RERANKER_PROVIDER}' — no local weights to fetch"
|
||||
fi
|
||||
|
||||
# Retry with exponential backoff for transient network failures, matching
|
||||
# the Docker build. Models land in the shared HF hub cache, so a re-run is
|
||||
# a fast cache validation rather than a re-download.
|
||||
local attempt delay=10
|
||||
for attempt in 1 2 3; do
|
||||
if PREWARM_EMB="$emb_model" PREWARM_RER="$rer_model" \
|
||||
HF_HUB_DOWNLOAD_TIMEOUT=600 \
|
||||
uv run --directory "$ROOT_DIR/hindsight-api-slim" python - <<'PY'
|
||||
import os
|
||||
|
||||
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "600")
|
||||
|
||||
import tiktoken
|
||||
|
||||
print(" caching tiktoken cl100k_base encoding...", flush=True)
|
||||
tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
emb = os.environ.get("PREWARM_EMB") or ""
|
||||
rer = os.environ.get("PREWARM_RER") or ""
|
||||
if emb:
|
||||
from sentence_transformers import SentenceTransformer
|
||||
|
||||
print(f" caching embedding model {emb} ...", flush=True)
|
||||
SentenceTransformer(emb)
|
||||
if rer:
|
||||
from sentence_transformers import CrossEncoder
|
||||
|
||||
print(f" caching cross-encoder model {rer} ...", flush=True)
|
||||
CrossEncoder(rer)
|
||||
print(" models cached", flush=True)
|
||||
PY
|
||||
then
|
||||
ok "model cache warmed"
|
||||
SUMMARY+=("models: cached for offline use")
|
||||
return
|
||||
fi
|
||||
if [ "$attempt" -lt 3 ]; then
|
||||
warn "download attempt $attempt failed — retrying in ${delay}s"
|
||||
sleep "$delay"
|
||||
delay=$((delay * 2))
|
||||
fi
|
||||
done
|
||||
warn "model pre-download failed after 3 attempts (network?) — models will download on first API use"
|
||||
SUMMARY+=("models: NOT cached (will download at runtime)")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Builds
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
build_ts_client() {
|
||||
step "Build TypeScript SDK (@vectorize-io/hindsight-client)"
|
||||
# The control plane imports the built SDK, so this must come first.
|
||||
if [ "$FORCE" = false ] && [ -d "$ROOT_DIR/hindsight-clients/typescript/dist" ]; then
|
||||
ok "dist/ present — skipping (use --force to rebuild)"
|
||||
return
|
||||
fi
|
||||
npm run build -w @vectorize-io/hindsight-client
|
||||
ok "SDK built"
|
||||
SUMMARY+=("ts-client: built")
|
||||
}
|
||||
|
||||
build_cli() {
|
||||
step "Build Hindsight CLI (cargo release)"
|
||||
# Building also vendors every crate into ~/.cargo, priming the offline cache.
|
||||
if [ "$FORCE" = false ] && [ -x "$ROOT_DIR/hindsight-cli/target/release/hindsight" ]; then
|
||||
ok "release binary present — skipping (use --force to rebuild)"
|
||||
return
|
||||
fi
|
||||
( cd "$ROOT_DIR/hindsight-cli" && cargo build --release )
|
||||
ok "CLI built (hindsight-cli/target/release/hindsight)"
|
||||
SUMMARY+=("cli: built")
|
||||
}
|
||||
|
||||
build_docs() {
|
||||
step "Build documentation site (Docusaurus)"
|
||||
INCLUDE_CURRENT_VERSION=true npm run build -w hindsight-docs
|
||||
ok "Docs built"
|
||||
SUMMARY+=("docs: built")
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Run
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
printf '\033[1mHindsight dev setup\033[0m (root: %s)\n' "$ROOT_DIR"
|
||||
[ "$SKIP_BUILD" = true ] && info "build steps disabled (--skip-build)"
|
||||
[ "$SKIP_MODELS" = true ] && info "model pre-download disabled (--skip-models)"
|
||||
[ "$FORCE" = true ] && info "force rebuild enabled (--force)"
|
||||
|
||||
ensure_uv
|
||||
ensure_node
|
||||
ensure_rust
|
||||
ensure_env_file
|
||||
setup_git_hooks
|
||||
|
||||
install_python_deps
|
||||
install_node_deps
|
||||
|
||||
[ "$SKIP_MODELS" = false ] && prewarm_models
|
||||
|
||||
if [ "$SKIP_BUILD" = false ]; then
|
||||
# NB: the control plane's Next.js production/standalone build is the Docker
|
||||
# deploy artifact and is intentionally NOT built here. Local dev runs it via
|
||||
# `npm run dev` (Turbopack, compiles on demand, no network needed offline).
|
||||
build_ts_client
|
||||
build_cli
|
||||
[ "$WITH_DOCS" = true ] && build_docs
|
||||
fi
|
||||
|
||||
step "Setup complete"
|
||||
for line in "${SUMMARY[@]}"; do ok "$line"; done
|
||||
cat <<'EOF'
|
||||
|
||||
Next steps:
|
||||
• Add your LLM API key to .env (HINDSIGHT_API_LLM_API_KEY)
|
||||
• Start everything: ./scripts/dev/start.sh
|
||||
• API only: ./scripts/dev/start-api.sh
|
||||
• Run API tests: cd hindsight-api-slim && uv run pytest tests/
|
||||
|
||||
The local embedding/reranker models and tiktoken encoding are cached, so the
|
||||
API can run fully offline (default local providers). Re-run with --skip-models
|
||||
to skip that step.
|
||||
EOF
|
||||
Reference in New Issue
Block a user