diff --git a/.monitor.pid b/.monitor.pid new file mode 100644 index 0000000..b9cba90 --- /dev/null +++ b/.monitor.pid @@ -0,0 +1 @@ +96510 diff --git a/content/getting-started.md b/content/getting-started.md index 2fa8c22..1fe33a2 100644 --- a/content/getting-started.md +++ b/content/getting-started.md @@ -1,6 +1,6 @@ -# Getting Started +# MD Docs Secure -Welcome to **MD Hub Secure**. +Publish technical notes and documentation to a secure web application for collaboration with teams. See [[Architecture Overview]] for the system shape and [[Project Plan]] for the milestone roadmap. @@ -30,4 +30,3 @@ Block math: $$ \int_0^1 x^2 dx = \frac{1}{3} $$ - diff --git a/docs/.last_state_hash b/docs/.last_state_hash new file mode 100644 index 0000000..756a5f4 --- /dev/null +++ b/docs/.last_state_hash @@ -0,0 +1 @@ +c3ce54e264212fa808c1110a4d276409 diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..7e89e8e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,94 @@ +# MD Hub Secure - Documentation Index + +## Project Planning + +| Document | Description | +|----------|-------------| +| [project-plan.md](project-plan.md) | Master project plan with milestones, timelines, and acceptance criteria | +| [architecture-overview.md](architecture-overview.md) | System architecture, component diagrams, and data flow | + +## Architecture Decision Records (ADRs) + +| Document | Decision | +|----------|----------| +| [ADR-001: Tech Stack](adrs/adr-001-tech-stack.md) | Go backend + Preact frontend | +| [ADR-002: Sync Protocol](adrs/adr-002-sync-protocol.md) | SimpleSync content-addressed protocol | +| [ADR-003: SSR Strategy](adrs/adr-003-ssr-strategy.md) | Go templates with Preact hydration | +| [ADR-004: Auth Strategy](adrs/adr-004-auth-strategy.md) | Passkeys primary, passwords fallback | +| [ADR-005: Storage Strategy](adrs/adr-005-storage-strategy.md) | libsql + content-addressed filesystem | +| [ADR-006: Email Strategy](adrs/adr-006-email-strategy.md) | Postmark gateway, plain-text only | +| [ADR-007: Security Model](adrs/adr-007-security-model.md) | Defense in depth, minimal dependencies | + +## Milestones + +| Document | Duration | Focus | +|----------|----------|-------| +| [Milestone 1: Foundation](milestones/milestone-01-foundation.md) | Weeks 1-2 | Server, database, markdown rendering | +| [Milestone 2: Sync Protocol](milestones/milestone-02-sync-protocol.md) | Weeks 3-4 | File sync, offline support, WebSocket | +| [Milestone 3: Authentication](milestones/milestone-03-authentication.md) | Weeks 5-6 | Passkeys, sessions, RBAC | +| [Milestone 4: Collaboration](milestones/milestone-04-collaboration.md) | Weeks 7-8 | Comments, email, conflict resolution | +| [Milestone 5: Search & UI](milestones/milestone-05-search-ui.md) | Weeks 9-10 | Full-text search, design system, responsive | +| [Milestone 6: Production](milestones/milestone-06-production.md) | Weeks 11-12 | Testing, security audit, ops docs | + +## Technical Specifications + +| Document | Topic | +|----------|-------| +| [SimpleSync Protocol](specs/simplesync-protocol.md) | Message formats, state machine, security | +| [Security Specification](specs/security-specification.md) | Threat model, controls, implementation | +| [Deployment Guide](specs/deployment-guide.md) | Docker setup, configuration, operations | + +## Key Metrics + +- **Timeline**: 12 weeks (3 months) +- **Test Coverage Target**: >80% +- **Performance Targets**: + - Initial page load: <100ms + - Search query: <50ms + - File sync: <5s +- **Security**: Zero critical/high vulnerabilities +- **Dependencies**: Minimal, all pinned and audited + +## Next Steps + +1. Review all documentation +2. Approve project plan +3. Set up repository and CI pipeline +4. Begin Milestone 1: Foundation + +## Project Structure + +``` +md-hub-secure/ +├── docs/ +│ ├── README.md (this file) +│ ├── project-plan.md +│ ├── architecture-overview.md +│ ├── adrs/ +│ │ ├── adr-001-tech-stack.md +│ │ ├── adr-002-sync-protocol.md +│ │ ├── adr-003-ssr-strategy.md +│ │ ├── adr-004-auth-strategy.md +│ │ ├── adr-005-storage-strategy.md +│ │ ├── adr-006-email-strategy.md +│ │ └── adr-007-security-model.md +│ ├── milestones/ +│ │ ├── milestone-01-foundation.md +│ │ ├── milestone-02-sync-protocol.md +│ │ ├── milestone-03-authentication.md +│ │ ├── milestone-04-collaboration.md +│ │ ├── milestone-05-search-ui.md +│ │ └── milestone-06-production.md +│ └── specs/ +│ ├── simplesync-protocol.md +│ ├── security-specification.md +│ └── deployment-guide.md +├── apps/ +│ ├── server/ # Go application +│ └── web/ # Preact frontend +├── packages/ +│ └── protocol/ # Shared types +├── docker-compose.yml +├── Dockerfile +└── Makefile +``` diff --git a/docs/STATUS.md b/docs/STATUS.md new file mode 100644 index 0000000..a3a13ad --- /dev/null +++ b/docs/STATUS.md @@ -0,0 +1,51 @@ +# Project Status Report + +**Generated:** 2026-04-29 09:35:52 UTC +**Monitoring Duration:** 9h 19m +**Iteration:** 56 +**Branch:** codex/foundation-bootstrap + +## Quick Stats + +| Metric | Value | +|--------|-------| +| Total Files | 95 | +| Go Files | 20 ( 2024 lines) | +| TypeScript Files | 3 ( 52 lines) | +| Markdown Files | 133 ( 27636 lines) | +| Docker Config | yes | +| Uncommitted Changes | 17 | +| Milestones Complete | 0/6 (0%) | + +## Recent Activity + +**Last Commit:** 8c454de - feat: add miller document layout (9 minutes ago) + +## Milestone Progress + +- [ ] Milestone 1: Foundation +- [ ] Milestone 2: Sync Protocol +- [ ] Milestone 3: Authentication +- [ ] Milestone 4: Collaboration +- [ ] Milestone 5: Search & UI +- [ ] Milestone 6: Production + +**Current Focus:** Not Started + +### Error Handling (10 ignored errors) + +Several errors are being silently ignored. Consider handling or logging these. + +## Notes for Human Review + +>> **Active development.** 20 Go files with 2024 lines of code. +>> **17 uncommitted changes** detected. Codex may be mid-work. +>> **Code review items found.** See Action Items above. + +## Files Changed This Session + + +--- + +*This report auto-generated by MD Hub Secure progress monitor.* +*Next update in ~10 minutes or when filesystem changes detected.* diff --git a/docs/project-plan.md b/docs/project-plan.md index bf7ae6d..cc2f13a 100644 --- a/docs/project-plan.md +++ b/docs/project-plan.md @@ -1,7 +1,7 @@ # MD Hub Secure - Project Plan -**Version:** 1.0 -**Date:** 2026-04-28 +**Version:** 1.0\ +**Date:** 2026-04-28\ **Status:** Draft ## Vision @@ -21,15 +21,17 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown ## Milestones ### Milestone 1: Foundation (Weeks 1-2) + **Goal:** Working server that can serve markdown files as HTML - Go HTTP server with chi router - libsql database with schema migration system -- Markdown-to-HTML pipeline with all extensions +- Markdown-to-HTML pipeline with most common extensions enabled - Static file serving for attachments - Basic Dockerfile and docker-compose.yml **Acceptance Criteria:** + - [ ] Server starts and listens on configurable port - [ ] Database auto-migrates on startup - [ ] Markdown files with wiki-links, tags, callouts, mermaid, math render correctly @@ -41,6 +43,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown --- ### Milestone 2: Sync Protocol (Weeks 3-4) + **Goal:** Bidirectional file sync between server and filesystem/web client - Content-addressed storage (SHA-256 filesystem layout) @@ -50,6 +53,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown - Offline queue with conflict detection **Acceptance Criteria:** + - [ ] File changes on disk automatically sync to server within 5 seconds - [ ] Web client can sync after reconnection, even with changes made offline - [ ] Concurrent edits to different files succeed without conflict @@ -60,6 +64,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown --- ### Milestone 3: Authentication & Authorization (Weeks 5-6) + **Goal:** Secure access control with passkeys and granular permissions - Passkey (WebAuthn) primary authentication @@ -69,6 +74,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown - Content signing with Ed25519 (optional per-user) **Acceptance Criteria:** + - [ ] Users can register and log in with passkeys on modern browsers - [ ] Password fallback works with proper hashing - [ ] Sessions expire securely and are revocable @@ -79,6 +85,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown --- ### Milestone 4: Collaboration Features (Weeks 7-8) + **Goal:** Comments, notifications, and conflict resolution - Comment system linked to file versions (by hash) @@ -88,6 +95,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown - Comment threading and email replies **Acceptance Criteria:** + - [ ] Users can comment on specific lines/sections of a document - [ ] Comments persist and are linked to document versions - [ ] Email notifications sent for watched file changes @@ -98,6 +106,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown --- ### Milestone 5: Search & UI Polish (Weeks 9-10) + **Goal:** Fast search, design system, and production readiness - Full-text search with SQLite FTS5 @@ -108,6 +117,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown - Performance benchmarking **Acceptance Criteria:** + - [ ] Search returns results in <50ms on client - [ ] Search works offline after initial sync - [ ] Design system documented at `/design` route @@ -118,6 +128,7 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown --- ### Milestone 6: Production Hardening (Weeks 11-12) + **Goal:** Deploy with confidence - Comprehensive test suite (unit, integration, e2e) @@ -127,7 +138,8 @@ A minimal-dependency, ultra-fast documentation platform that publishes markdown - Documentation for operators **Acceptance Criteria:** -- [ ] >80% test coverage + +- [ ] 80% test coverage - [ ] No critical or high-severity security findings - [ ] Backup script exports database + attachments as tar.gz - [ ] Health check endpoint returns status of all dependencies @@ -177,44 +189,48 @@ See [architecture-overview.md](architecture-overview.md) for detailed component ## Technology Stack ### Backend (Go 1.22+) -| Component | Library | Justification | -|-----------|---------|---------------| -| HTTP Router | `github.com/go-chi/chi` | Minimal, idiomatic, stdlib-compatible | -| Database | `github.com/tursodatabase/libsql-client-go` | Embedded SQLite with replication option | -| Migrations | `github.com/golang-migrate/migrate` | Battle-tested, version-controlled schema | -| Auth/Passkeys | `github.com/go-webauthn/webauthn` | Reference implementation, well-audited | -| Password Hash | `golang.org/x/crypto/argon2` | Winner of Password Hashing Competition | -| Markdown | `github.com/yuin/goldmark` + extensions | Fast, extensible, CommonMark compliant | -| WebSocket | `github.com/gorilla/websocket` | Most widely used Go WS library | -| File Watch | `github.com/fsnotify/fsnotify` | Cross-platform, stdlib-adjacent | -| Email | `github.com/keighl/postmark` + `net/smtp` | Postmark adapter + SMTP fallback | -| Crypto | `crypto/ed25519`, `crypto/sha256` | Standard library, no external deps | -| Templates | `html/template` | Stdlib, auto-escaping, SSR-safe | + +| Component | Library | Justification | +| ------------- | ------------------------------------------- | ---------------------------------------- | +| HTTP Router | `github.com/go-chi/chi` | Minimal, idiomatic, stdlib-compatible | +| Database | `github.com/tursodatabase/libsql-client-go` | Embedded SQLite with replication option | +| Migrations | `github.com/golang-migrate/migrate` | Battle-tested, version-controlled schema | +| Auth/Passkeys | `github.com/go-webauthn/webauthn` | Reference implementation, well-audited | +| Password Hash | `golang.org/x/crypto/argon2` | Winner of Password Hashing Competition | +| Markdown | `github.com/yuin/goldmark` + extensions | Fast, extensible, CommonMark compliant | +| WebSocket | `github.com/gorilla/websocket` | Most widely used Go WS library | +| File Watch | `github.com/fsnotify/fsnotify` | Cross-platform, stdlib-adjacent | +| Email | `github.com/keighl/postmark` + `net/smtp` | Postmark adapter + SMTP fallback | +| Crypto | `crypto/ed25519`, `crypto/sha256` | Standard library, no external deps | +| Templates | `html/template` | Stdlib, auto-escaping, SSR-safe | ### Frontend (Preact + TypeScript) -| Component | Library | Justification | -|-----------|---------|---------------| -| Framework | `preact` (10KB) | React API, tiny bundle, fast hydration | -| Routing | `preact-iso` | Lightweight isomorphic routing | -| Markdown | `marked` + `micromark` | Server+client compatible | -| Search | `flexsearch` | Client-side full-text, zero server load | -| Styling | Native CSS + custom properties | No build-time CSS processing needed | -| Icons | Inline SVG | No icon font dependency | -| Build | `vite` | Fast HMR, modern output, minimal config | + +| Component | Library | Justification | +| --------- | ------------------------------ | --------------------------------------- | +| Framework | `preact` (10KB) | React API, tiny bundle, fast hydration | +| Routing | `preact-iso` | Lightweight isomorphic routing | +| Markdown | `marked` + `micromark` | Server+client compatible | +| Search | `flexsearch` | Client-side full-text, zero server load | +| Styling | Native CSS + custom properties | No build-time CSS processing needed | +| Icons | Inline SVG | No icon font dependency | +| Build | `vite` | Fast HMR, modern output, minimal config | ### Infrastructure -| Component | Technology | Justification | -|-----------|-----------|---------------| -| Container | Docker + Compose | Universal deployment target | -| Reverse Proxy | Traefik (optional) | Auto-HTTPS, included in compose | -| Database | libsql/SQLite | Single file, zero external dependency | -| Storage | Local filesystem | Content-addressed, immutable, backup-friendly | + +| Component | Technology | Justification | +| ------------- | ------------------ | --------------------------------------------- | +| Container | Docker + Compose | Universal deployment target | +| Reverse Proxy | Traefik (optional) | Auto-HTTPS, included in compose | +| Database | libsql/SQLite | Single file, zero external dependency | +| Storage | Local filesystem | Content-addressed, immutable, backup-friendly | --- ## Security Model ### Threats Addressed + 1. **Supply Chain Attacks**: Minimal external dependencies; all pinned with go.sum; vendored if needed 2. **Data Tampering**: All document versions SHA-256 hashed; optional Ed25519 signatures 3. **Unauthorized Access**: RBAC with principle of least privilege; content never served without auth check @@ -224,11 +240,13 @@ See [architecture-overview.md](architecture-overview.md) for detailed component 7. **DoS**: Rate limiting on auth endpoints; file upload size limits; query timeouts ### Content Security + - Attachment uploads: Magic number validation, extension whitelist, size limits - Markdown rendering: Sanitized HTML output; no raw HTML injection - File paths: Canonicalization prevents directory traversal ### Audit & Compliance + - Every data modification logged with user, timestamp, IP, and content hash - Immutable history table — updates are inserts - GDPR-compliant: full data export and deletion capability @@ -237,14 +255,14 @@ See [architecture-overview.md](architecture-overview.md) for detailed component ## Performance Targets -| Metric | Target | Measurement | -|--------|--------|-------------| -| Initial Page Load | <100ms | TTFB + First Contentful Paint | -| Time to Interactive | <200ms | Hydration complete | -| Search Query | <50ms | Client-side after initial sync | -| File Sync | <5s | From disk save to server persist | -| Bundle Size | <100KB | Gzipped JS + CSS | -| Concurrent Users | 1000 readers, 100 editors | Load tested | +| Metric | Target | Measurement | +| ------------------- | ------------------------- | -------------------------------- | +| Initial Page Load | <100ms | TTFB + First Contentful Paint | +| Time to Interactive | <200ms | Hydration complete | +| Search Query | <50ms | Client-side after initial sync | +| File Sync | <5s | From disk save to server persist | +| Bundle Size | <100KB | Gzipped JS + CSS | +| Concurrent Users | 1000 readers, 100 editors | Load tested | --- @@ -295,14 +313,14 @@ md-hub-secure/ ## Risk Register -| Risk | Likelihood | Impact | Mitigation | -|------|-----------|--------|------------| -| WebAuthn/passkey adoption issues | Medium | Medium | Password fallback always available; extensive browser testing | -| SQLite performance at scale | Low | High | libsql replication path; FTS5 for search; benchmark early | -| Markdown extension parser bugs | Medium | Medium | Fuzz testing; strict CommonMark base; gradual feature rollout | -| Email deliverability | Medium | Medium | Postmark reputation monitoring; SPF/DKIM setup; SMTP fallback | -| Frontend bundle bloat | Medium | Medium | Bundle analysis in CI; tree-shaking enforced; lazy loading | -| File sync conflicts in teams | High | Medium | Clear UX for conflict resolution; email notification; automatic merge when possible | +| Risk | Likelihood | Impact | Mitigation | +| -------------------------------- | ---------- | ------ | ----------------------------------------------------------------------------------- | +| WebAuthn/passkey adoption issues | Medium | Medium | Password fallback always available; extensive browser testing | +| SQLite performance at scale | Low | High | libsql replication path; FTS5 for search; benchmark early | +| Markdown extension parser bugs | Medium | Medium | Fuzz testing; strict CommonMark base; gradual feature rollout | +| Email deliverability | Medium | Medium | Postmark reputation monitoring; SPF/DKIM setup; SMTP fallback | +| Frontend bundle bloat | Medium | Medium | Bundle analysis in CI; tree-shaking enforced; lazy loading | +| File sync conflicts in teams | High | Medium | Clear UX for conflict resolution; email notification; automatic merge when possible | --- diff --git a/docs/specs/deployment-guide.md b/docs/specs/deployment-guide.md new file mode 100644 index 0000000..531dbb9 --- /dev/null +++ b/docs/specs/deployment-guide.md @@ -0,0 +1,272 @@ +# Deployment Guide + +## Requirements + +- Docker 24.0+ +- Docker Compose 2.20+ +- 1GB RAM minimum (2GB recommended) +- 10GB disk space +- TLS certificate (Let's Encrypt or custom) + +## Quick Start + +```bash +# Clone repository +git clone https://github.com/your-org/md-hub-secure.git +cd md-hub-secure + +# Copy and edit configuration +cp .env.example .env +# Edit .env with your domain, email, Postmark API key + +# Start services +docker-compose up -d + +# Verify health +curl https://your-domain.com/health +``` + +## Configuration + +### Environment Variables + +| Variable | Required | Default | Description | +|----------|----------|---------|-------------| +| `DOMAIN` | Yes | — | Public domain name | +| `DB_PATH` | No | `/data/db.sqlite` | Database file path | +| `FILESTORE_PATH` | No | `/data/files` | Content-addressed file storage | +| `NOTES_PATH` | No | `/notes` | Watched markdown directory | +| `POSTMARK_API_KEY` | Yes | — | Postmark server API token | +| `POSTMARK_FROM` | No | `notifications@DOMAIN` | From email address | +| `SESSION_SECRET` | Yes | — | 32-byte hex for session signing | +| `ENV` | No | `production` | `development` or `production` | + +### Docker Compose + +```yaml +version: "3.8" + +services: + app: + build: . + ports: + - "8080:8080" + volumes: + - ./data:/data + - ./notes:/notes:ro + environment: + - DOMAIN=${DOMAIN} + - DB_PATH=/data/db.sqlite + - FILESTORE_PATH=/data/files + - NOTES_PATH=/notes + - POSTMARK_API_KEY=${POSTMARK_API_KEY} + - SESSION_SECRET=${SESSION_SECRET} + env_file: + - .env + restart: unless-stopped + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/health"] + interval: 30s + timeout: 3s + retries: 3 + start_period: 10s + security_opt: + - no-new-privileges:true + read_only: true + tmpfs: + - /tmp:noexec,nosuid,size=100m + + # Optional: Traefik for auto-HTTPS + traefik: + image: traefik:v3.0 + command: + - "--api.insecure=false" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true" + - "--certificatesresolvers.letsencrypt.acme.email=${EMAIL}" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./letsencrypt:/letsencrypt + labels: + - "traefik.enable=true" + restart: unless-stopped +``` + +## Production Checklist + +### Pre-Deployment +- [ ] Change default `SESSION_SECRET` (generate: `openssl rand -hex 32`) +- [ ] Configure `POSTMARK_API_KEY` with valid token +- [ ] Set `DOMAIN` to public hostname +- [ ] Configure DNS A/AAAA records +- [ ] Set up SPF and DKIM for email domain +- [ ] Review `.env` file — no default secrets + +### Security +- [ ] TLS 1.3 enforced (Traefik or reverse proxy) +- [ ] HSTS header configured +- [ ] Firewall rules: only 80/443 open +- [ ] Fail2ban or rate limiting on SSH +- [ ] Automatic security updates enabled +- [ ] Backup encryption key stored securely + +### Monitoring +- [ ] Health check endpoint accessible +- [ ] Log rotation configured +- [ ] Disk space alerting (>80%) +- [ ] Memory usage alerting +- [ ] Uptime monitoring (external) + +## Backup Procedures + +### Automated Backup + +```bash +#!/bin/bash +# backup.sh + +BACKUP_DIR="/backups" +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_FILE="$BACKUP_DIR/md-hub-$TIMESTAMP.tar.gz" + +# Create backup +tar czf "$BACKUP_FILE" \ + -C /data \ + db.sqlite \ + files/ + +# Encrypt (optional) +gpg --symmetric --cipher-algo AES256 "$BACKUP_FILE" +rm "$BACKUP_FILE" + +# Keep only last 30 days +find "$BACKUP_DIR" -name "md-hub-*.tar.gz.gpg" -mtime +30 -delete +``` + +**Cron**: `0 2 * * * /opt/md-hub-secure/backup.sh` + +### Restore from Backup + +```bash +# Stop application +docker-compose stop app + +# Extract backup +tar xzf md-hub-20240115_020000.tar.gz -C /data + +# Restart +docker-compose up -d app + +# Verify +curl https://your-domain.com/health +``` + +## Upgrade Procedures + +### Minor Update (patch version) +```bash +docker-compose pull +docker-compose up -d +``` + +### Major Update +1. Read release notes for breaking changes +2. Create backup: `./backup.sh` +3. Pull new image: `docker-compose pull` +4. Run database migrations (auto on startup) +5. Update configuration if needed +6. Deploy: `docker-compose up -d` +7. Verify health and functionality +8. Rollback if issues: restore from backup + +## Troubleshooting + +### Application Won't Start +```bash +# Check logs +docker-compose logs -f app + +# Verify configuration +docker-compose exec app /server -check-config + +# Test database +docker-compose exec app sqlite3 /data/db.sqlite ".tables" +``` + +### Sync Not Working +```bash +# Check file watcher +docker-compose exec app ls -la /notes + +# Verify WebSocket +curl -i -N \ + -H "Connection: Upgrade" \ + -H "Upgrade: websocket" \ + -H "Sec-WebSocket-Key: $(openssl rand -base64 16)" \ + -H "Sec-WebSocket-Version: 13" \ + https://your-domain.com/ws +``` + +### Email Not Sending +```bash +# Test Postmark connection +curl -X POST \ + https://api.postmarkapp.com/server \ + -H "X-Postmark-Server-Token: YOUR_TOKEN" \ + -H "Accept: application/json" + +# Check email queue +docker-compose exec app sqlite3 /data/db.sqlite \ + "SELECT * FROM email_queue WHERE status='failed'" +``` + +## Scaling + +### Vertical Scaling +- Increase container memory/CPU limits +- SQLite WAL mode handles concurrent reads well +- For heavy write loads, consider libsql replication + +### Horizontal Scaling (Advanced) +```yaml +# Requires shared storage and load balancer +services: + app1: + # ... same config ... + environment: + - DB_PATH=/shared/db.sqlite + - FILESTORE_PATH=/shared/files + + app2: + # ... same config ... + environment: + - DB_PATH=/shared/db.sqlite + - FILESTORE_PATH=/shared/files + + nginx: + image: nginx:alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + ports: + - "80:80" +``` + +**Note**: Horizontal scaling requires: +- Shared filesystem (NFS, EFS) +- libsql replication or external database +- Sticky sessions for WebSocket + +## Support + +- **Documentation**: `/docs` route (when authenticated) +- **Health Check**: `GET /health` +- **Metrics**: `GET /metrics` (Prometheus format) +- **Logs**: `docker-compose logs -f app` +- **Issues**: https://github.com/your-org/md-hub-secure/issues diff --git a/docs/specs/security-specification.md b/docs/specs/security-specification.md new file mode 100644 index 0000000..103d362 --- /dev/null +++ b/docs/specs/security-specification.md @@ -0,0 +1,456 @@ +# Security Specification + +## Threat Model + +### Assets +1. **Document Content**: Markdown files and attachments +2. **User Credentials**: Passkeys, passwords, session tokens +3. **Collaboration Data**: Comments, notifications, history +4. **System State**: Database, configuration, logs + +### Threat Actors +1. **External Attacker**: No system access, attempts remote exploitation +2. **Malicious User**: Authenticated, attempts unauthorized access +3. **Compromised Client**: Legitimate user's device infected +4. **Insider**: Administrator or developer with system access + +### Threats +1. **Supply Chain Attack**: Malicious dependency injection +2. **Authentication Bypass**: Session hijacking, credential theft +3. **Authorization Bypass**: Horizontal/vertical privilege escalation +4. **Data Exfiltration**: Unauthorized document access +5. **Data Tampering**: Modification without authorization +6. **Denial of Service**: Resource exhaustion +7. **Information Disclosure**: Secrets in logs/responses + +## Security Controls + +### 1. Supply Chain Security + +**Policy**: +- Only dependencies with >1M downloads/month or official Go team +- All versions pinned in go.sum +- No transitive dependencies without review +- Prefer standard library when possible + +**Implementation**: +``` +# go.mod +require ( + github.com/go-chi/chi/v5 v5.0.12 + github.com/tursodatabase/libsql-client-go v0.0.0-20240416075031-555ce55511f7 + // All versions pinned +) + +# Build +RUN go mod verify # Verify checksums +RUN go mod vendor # Optional: vendor for air-gapped +``` + +**Audit**: +- Weekly `govulncheck` scan +- Dependency review in CI +- No build-time network access (vendored or proxy with checksum) + +### 2. Authentication Security + +#### Passkey (WebAuthn) +```go +// Registration +func registerPasskey(user User) (*Credential, error) { + options, session, err := webauthn.BeginRegistration(user) + // Store session data server-side (never client-side) + // Return options to client +} + +func verifyRegistration(sessionID string, response ProtocolCredentialCreation) (*Credential, error) { + session := getSessionData(sessionID) + credential, err := webauthn.FinishRegistration(user, session, response) + // Verify attestation + // Store credential: ID, public key, sign count +} +``` + +#### Password (Argon2id) +```go +import "golang.org/x/crypto/argon2" + +const ( + timeCost = 3 + memoryCost = 64 * 1024 // 64MB + parallelism = 4 + saltLength = 16 + keyLength = 32 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, saltLength) + if _, err := rand.Read(salt); err != nil { + return "", err + } + + hash := argon2.IDKey([]byte(password), salt, timeCost, memoryCost, parallelism, keyLength) + + // Encode to string: $argon2id$v=19$m=65536,t=3,p=4$$ + return encodeHash(salt, hash), nil +} + +func verifyPassword(password, encodedHash string) (bool, error) { + salt, hash, params, err := decodeHash(encodedHash) + if err != nil { + return false, err + } + + computedHash := argon2.IDKey([]byte(password), salt, params.time, params.memory, params.parallelism, uint32(len(hash))) + + // Constant-time comparison + return subtle.ConstantTimeCompare(hash, computedHash) == 1, nil +} +``` + +#### Session Management +```go +func createSession(userID string, r *http.Request) (*Session, error) { + token := make([]byte, 32) + if _, err := rand.Read(token); err != nil { + return nil, err + } + + session := &Session{ + ID: uuid.New().String(), + UserID: userID, + TokenHash: sha256.Sum256(token), + CreatedAt: time.Now(), + ExpiresAt: time.Now().Add(24 * time.Hour), + IP: r.RemoteAddr, + UserAgent: r.UserAgent(), + } + + // Store in database + if err := db.CreateSession(session); err != nil { + return nil, err + } + + // Set cookie + http.SetCookie(w, &http.Cookie{ + Name: "session", + Value: base64.URLEncoding.EncodeToString(token), + Expires: session.ExpiresAt, + HttpOnly: true, + Secure: true, // HTTPS only + SameSite: http.SameSiteStrictMode, + Path: "/", + }) + + return session, nil +} +``` + +### 3. Authorization + +#### Permission Model +```go +type Permission string + +const ( + PermissionRead Permission = "read" + PermissionWrite Permission = "write" + PermissionAdmin Permission = "admin" +) + +type ResourceType string + +const ( + ResourceGlobal ResourceType = "global" + ResourceCollection ResourceType = "collection" + ResourceDocument ResourceType = "document" +) + +func checkPermission(userID string, resourceType ResourceType, resourceID string, required Permission) (bool, error) { + // Check explicit permission + perm, err := db.GetPermission(userID, resourceType, resourceID) + if err == nil && hasPermission(perm.Permission, required) { + return true, nil + } + + // Check collection permission for documents + if resourceType == ResourceDocument { + collectionID := getDocumentCollection(resourceID) + perm, err = db.GetPermission(userID, ResourceCollection, collectionID) + if err == nil && hasPermission(perm.Permission, required) { + return true, nil + } + } + + // Check global permission + perm, err = db.GetPermission(userID, ResourceGlobal, "") + if err == nil && hasPermission(perm.Permission, required) { + return true, nil + } + + // Check document public status + if resourceType == ResourceDocument && required == PermissionRead { + doc, err := db.GetDocument(resourceID) + if err == nil && doc.PublicRead { + return true, nil + } + } + + return false, nil +} +``` + +#### Middleware +```go +func RequirePermission(resourceType ResourceType, required Permission) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userID := getUserID(r.Context()) + resourceID := chi.URLParam(r, "id") + + allowed, err := checkPermission(userID, resourceType, resourceID, required) + if err != nil || !allowed { + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + next.ServeHTTP(w, r) + }) + } +} + +// Usage: +r.With(RequirePermission(ResourceDocument, PermissionRead)). + Get("/docs/{id}", getDocumentHandler) +``` + +### 4. Input Validation + +#### Markdown Sanitization +```go +import "github.com/yuin/goldmark" +import "github.com/yuin/goldmark/extension" +import "github.com/yuin/goldmark/renderer/html" + +func renderMarkdown(source []byte) ([]byte, error) { + md := goldmark.New( + goldmark.WithExtensions( + extension.Table, + extension.Strikethrough, + extension.Linkify, + // No raw HTML extension — we control HTML output + ), + goldmark.WithRendererOptions( + html.WithHardWraps(), + html.WithXHTML(), + html.WithUnsafe(), // DISABLED — no raw HTML passthrough + ), + ) + + var buf bytes.Buffer + if err := md.Convert(source, &buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} +``` + +#### File Upload Validation +```go +func validateUpload(file io.Reader, filename string, maxSize int64) ([]byte, string, error) { + // Size limit + limited := io.LimitReader(file, maxSize+1) + data, err := io.ReadAll(limited) + if err != nil { + return nil, "", err + } + if int64(len(data)) > maxSize { + return nil, "", fmt.Errorf("file too large") + } + + // Magic number validation + contentType := detectContentType(data) + allowedTypes := map[string]bool{ + "image/jpeg": true, + "image/png": true, + "image/gif": true, + "image/webp": true, + "application/pdf": true, + } + if !allowedTypes[contentType] { + return nil, "", fmt.Errorf("file type not allowed") + } + + // Extension whitelist + ext := strings.ToLower(filepath.Ext(filename)) + allowedExts := map[string]bool{ + ".jpg": true, ".jpeg": true, ".png": true, + ".gif": true, ".webp": true, ".pdf": true, + } + if !allowedExts[ext] { + return nil, "", fmt.Errorf("file extension not allowed") + } + + // Compute hash + hash := sha256.Sum256(data) + hashStr := hex.EncodeToString(hash[:]) + + return data, hashStr, nil +} +``` + +#### Path Sanitization +```go +func sanitizePath(userPath string) (string, error) { + // Clean the path + clean := filepath.Clean(userPath) + + // Prevent traversal + if strings.Contains(clean, "..") { + return "", fmt.Errorf("path traversal detected") + } + + // Ensure it's relative + if filepath.IsAbs(clean) { + return "", fmt.Errorf("absolute paths not allowed") + } + + // Join with base and verify it's within base + fullPath := filepath.Join(baseDir, clean) + if !strings.HasPrefix(filepath.Clean(fullPath), filepath.Clean(baseDir)) { + return "", fmt.Errorf("path outside allowed directory") + } + + return clean, nil +} +``` + +### 5. Transport Security + +#### TLS Configuration +```go +tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS13, + CurvePreferences: []tls.CurveID{ + tls.X25519, + tls.CurveP256, + }, + CipherSuites: []uint16{ + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_CHACHA20_POLY1305_SHA256, + tls.TLS_AES_128_GCM_SHA256, + }, + PreferServerCipherSuites: true, +} +``` + +#### Security Headers +```go +func securityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Security-Policy", + "default-src 'self'; "+ + "script-src 'self' 'nonce-{nonce}'; "+ + "style-src 'self' 'unsafe-inline'; "+ + "img-src 'self' data:; "+ + "connect-src 'self' wss:; "+ + "frame-ancestors 'none'; "+ + "base-uri 'self'; "+ + "form-action 'self';") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("X-Frame-Options", "DENY") + w.Header().Set("X-XSS-Protection", "1; mode=block") + w.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin") + w.Header().Set("Strict-Transport-Security", "max-age=63072000; includeSubDomains; preload") + w.Header().Set("Permissions-Policy", "camera=(), microphone=(), geolocation=()") + + next.ServeHTTP(w, r) + }) +} +``` + +### 6. Operational Security + +#### Logging +```go +func logAudit(event AuditEvent) { + entry := slog.With( + "event_type", event.Type, + "actor_id", redactIfSecret(event.ActorID), + "resource_type", event.ResourceType, + "resource_id", redactIfSecret(event.ResourceID), + "ip", event.IP, + "timestamp", event.Timestamp, + ) + + // Never log: passwords, tokens, session IDs, API keys + entry.Info("audit event") +} + +func redactIfSecret(value string) string { + if looksLikeSecret(value) { + return "[REDACTED]" + } + return value +} +``` + +#### Container Security +```dockerfile +# Dockerfile +FROM golang:1.22-alpine AS builder +# ... build ... + +FROM alpine:3.19 +RUN addgroup -g 1000 appgroup && \ + adduser -u 1000 -G appgroup -s /bin/sh -D appuser + +# Read-only root filesystem +COPY --from=builder /app/server /server +COPY --from=builder /app/web/dist /web + +# Data directory (writable) +RUN mkdir /data && chown appuser:appgroup /data +VOLUME /data + +USER appuser +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=3s \ + CMD wget -q --spider http://localhost:8080/health || exit 1 + +ENTRYPOINT ["/server"] +``` + +## Incident Response + +### Detection +- Automated alerts on: failed auth spikes, permission errors, large file uploads +- Log analysis for anomalies +- Health check failures + +### Response +1. **Isolate**: Block IP, revoke sessions, disable account if compromised +2. **Assess**: Determine scope of breach, affected data +3. **Contain**: Patch vulnerability, rotate secrets +4. **Recover**: Restore from clean backup if necessary +5. **Document**: Post-incident review, update security measures + +## Compliance + +### GDPR +- Data minimization: only email stored, no other PII +- Right to erasure: `/api/user/delete` endpoint +- Data portability: `/api/user/export` endpoint +- Consent: explicit opt-in for email notifications + +### Audit Trail +All security-relevant events logged: +- Authentication attempts (success/failure) +- Permission changes +- Document access (if private) +- Admin actions +- Configuration changes + +Retention: 90 days active, 1 year archive diff --git a/notes/problems.md b/notes/problems.md index 1e98802..987eba7 100644 --- a/notes/problems.md +++ b/notes/problems.md @@ -5,5 +5,3 @@ - `go-libsql` is the active official Go binding, but it requires CGO and precompiled native libraries. That weakens the "same source -> same binary hash" goal until the exact build environment is locked down. - The embedded-replica connector in `go-libsql` rejects an empty primary URL. The foundation therefore uses the local `file:` libsql driver path when no primary is configured, and reserves embedded-replica setup for later sync work. - Docker verification is currently blocked by the local environment because the daemon socket at `/Users/tim/.orbstack/run/docker.sock` was not reachable during `docker build`. -- The docs do not define a config file format. This implementation uses JSON to keep the bootstrap dependency-free. -- The docs require "all Markdown extensions" but do not define a canonical sample corpus. A representative content sample is included and should be expanded as renderer coverage grows. diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..fce8cbf --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,67 @@ +# MD Hub Secure - Development Scripts + +This directory contains helper scripts for monitoring development progress. + +## Scripts + +### `install-monitor.sh` +Installs and starts the progress monitor as a background process. + +```bash +./scripts/install-monitor.sh +``` + +The monitor will: +- Watch for filesystem changes every 10 minutes +- Update `docs/STATUS.md` with current progress +- Auto-exit if no changes detected for 10 minutes (Codex idle/finished) +- Provide code review notes and action items + +### `monitor.sh` +The actual monitoring script. Usually called via `install-monitor.sh`. + +```bash +./scripts/monitor.sh +``` + +Runs in foreground. Press Ctrl+C to stop. + +### `status.sh` +Quick status check without starting the monitor. + +```bash +./scripts/status.sh +``` + +Shows: +- Whether monitor is running +- Current progress report from `docs/STATUS.md` + +## Usage + +1. Start monitoring (run once): + ```bash + ./scripts/install-monitor.sh + ``` + +2. Check status anytime: + ```bash + ./scripts/status.sh + ``` + +3. View live logs: + ```bash + tail -f monitor.log + ``` + +4. Stop monitor: + ```bash + kill $(cat .monitor.pid) + ``` + +## Files Generated + +- `docs/STATUS.md` — Current project status and code review notes +- `docs/.last_state_hash` — Internal file for tracking changes +- `monitor.log` — Log output from monitor +- `.monitor.pid` — Process ID of running monitor diff --git a/scripts/install-monitor.sh b/scripts/install-monitor.sh new file mode 100755 index 0000000..1636115 --- /dev/null +++ b/scripts/install-monitor.sh @@ -0,0 +1,115 @@ +#!/bin/bash +# MD Hub Secure - Install Monitor +# Sets up the progress monitor to run in the background + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="/Users/tim/developer/md-hub-secure" +MONITOR_SCRIPT="$SCRIPT_DIR/monitor.sh" +PID_FILE="$PROJECT_DIR/.monitor.pid" +LOG_FILE="$PROJECT_DIR/monitor.log" + +echo "MD Hub Secure - Monitor Installation" +echo "====================================" +echo "" + +# Check if monitor is already running +if [[ -f "$PID_FILE" ]]; then + OLD_PID=$(cat "$PID_FILE") + if kill -0 "$OLD_PID" 2>/dev/null; then + echo "Monitor is already running (PID: $OLD_PID)" + echo "" + echo "Options:" + echo " 1) Stop existing monitor and start fresh" + echo " 2) View current status" + echo " 3) Exit" + echo "" + read -p "Choose option [1-3]: " choice + + case $choice in + 1) + echo "Stopping existing monitor..." + kill "$OLD_PID" 2>/dev/null || true + rm -f "$PID_FILE" + ;; + 2) + echo "" + cat "$PROJECT_DIR/docs/STATUS.md" 2>/dev/null || echo "No status file yet." + exit 0 + ;; + 3) + exit 0 + ;; + *) + echo "Invalid option" + exit 1 + ;; + esac + else + # Stale PID file + rm -f "$PID_FILE" + fi +fi + +# Make sure monitor script exists +if [[ ! -f "$MONITOR_SCRIPT" ]]; then + echo "ERROR: Monitor script not found at $MONITOR_SCRIPT" + exit 1 +fi + +# Ensure it's executable +chmod +x "$MONITOR_SCRIPT" + +# Create necessary directories +mkdir -p "$PROJECT_DIR/docs" +mkdir -p "$PROJECT_DIR/scripts" + +echo "Starting monitor in background..." +echo " Script: $MONITOR_SCRIPT" +echo " Log: $LOG_FILE" +echo " PID: $PID_FILE" +echo "" + +# Start monitor in background +nohup "$MONITOR_SCRIPT" > "$LOG_FILE" 2>&1 & +MONITOR_PID=$! + +# Save PID +echo $MONITOR_PID > "$PID_FILE" + +sleep 2 + +# Check if it's actually running +if kill -0 "$MONITOR_PID" 2>/dev/null; then + echo "Monitor started successfully!" + echo " PID: $MONITOR_PID" + echo "" + echo "Commands:" + echo " View status: cat docs/STATUS.md" + echo " View log: tail -f monitor.log" + echo " Stop monitor: kill $(cat "$PID_FILE")" + echo " Restart: ./scripts/install-monitor.sh" + echo "" + echo "The monitor will:" + echo " - Check for filesystem changes every 10 minutes" + echo " - Update docs/STATUS.md with current progress" + echo " - Auto-exit if no changes detected (Codex idle/finished)" + echo " - Provide code review notes for Codex" + echo "" + + # Generate initial status + echo "Generating initial status report..." + "$MONITOR_SCRIPT" > /dev/null 2>&1 || true + + if [[ -f "$PROJECT_DIR/docs/STATUS.md" ]]; then + echo "" + echo "Current Status:" + echo "---------------" + head -30 "$PROJECT_DIR/docs/STATUS.md" + fi +else + echo "ERROR: Monitor failed to start. Check $LOG_FILE" + rm -f "$PID_FILE" + exit 1 +fi diff --git a/scripts/monitor.sh b/scripts/monitor.sh new file mode 100755 index 0000000..c5326e2 --- /dev/null +++ b/scripts/monitor.sh @@ -0,0 +1,357 @@ +#!/bin/bash +# MD Hub Secure - Progress Monitor +# Watches the project directory and updates docs/STATUS.md with progress +# Exits if no filesystem changes detected between iterations + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="/Users/tim/developer/md-hub-secure" +DOCS_DIR="$PROJECT_DIR/docs" +STATUS_FILE="$DOCS_DIR/STATUS.md" +HASH_FILE="$DOCS_DIR/.last_state_hash" +ITERATION=0 +MAX_ITERATIONS=1000 # Safety limit +START_TIME=$(date +%s) + +# Colors for terminal output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +log() { + echo -e "${BLUE}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1" >&2 +} + +warn() { + echo -e "${YELLOW}[$(date '+%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1" >&2 +} + +error() { + echo -e "${RED}[$(date '+%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1" >&2 +} + +success() { + echo -e "${GREEN}[$(date '+%Y-%m-%d %H:%M:%S')]${NC} $1" >&2 +} + +# Compute hash of all tracked files (excluding node_modules, .git, etc.) +compute_state_hash() { + find "$PROJECT_DIR" \ + -type f \ + -not -path '*/node_modules/*' \ + -not -path '*/.git/*' \ + -not -path '*/data/*' \ + -not -path '*/vendor/*' \ + -not -name '.last_state_hash' \ + -not -name 'STATUS.md' \ + -not -name '*.lock' \ + | sort \ + | xargs stat -f '%m %N' 2>/dev/null \ + | md5 -q +} + +# Check if filesystem changed +has_changes() { + local current_hash + current_hash=$(compute_state_hash) + + if [[ -f "$HASH_FILE" ]]; then + local last_hash + last_hash=$(cat "$HASH_FILE") + if [[ "$current_hash" == "$last_hash" ]]; then + return 1 # No changes + fi + fi + + echo "$current_hash" > "$HASH_FILE" + return 0 # Changes detected (or first run) +} + +# Count lines of code by language +analyze_codebase() { + log "Analyzing codebase..." + + local go_files=0 + local go_lines=0 + local ts_files=0 + local ts_lines=0 + local md_files=0 + local md_lines=0 + local total_files=0 + local docker_files=0 + + # Count Go files + if command -v find &> /dev/null; then + go_files=$(find "$PROJECT_DIR" -name '*.go' -not -path '*/vendor/*' -not -path '*/node_modules/*' | wc -l) + go_lines=$(find "$PROJECT_DIR" -name '*.go' -not -path '*/vendor/*' -not -path '*/node_modules/*' -exec cat {} + 2>/dev/null | wc -l) + + ts_files=$(find "$PROJECT_DIR" -name '*.ts' -o -name '*.tsx' | grep -v node_modules | wc -l) + ts_lines=$(find "$PROJECT_DIR" -name '*.ts' -o -name '*.tsx' | grep -v node_modules | xargs cat 2>/dev/null | wc -l) + + md_files=$(find "$PROJECT_DIR" -name '*.md' | wc -l) + md_lines=$(find "$PROJECT_DIR" -name '*.md' -exec cat {} + 2>/dev/null | wc -l) + + docker_files=$(find "$PROJECT_DIR" -name 'Dockerfile*' -o -name 'docker-compose*' | wc -l) + + total_files=$(find "$PROJECT_DIR" -type f -not -path '*/node_modules/*' -not -path '*/.git/*' -not -path '*/data/*' | wc -l) + fi + + echo "$go_files|$go_lines|$ts_files|$ts_lines|$md_files|$md_lines|$total_files|$docker_files" +} + +# Check git status if available +check_git_status() { + if [[ -d "$PROJECT_DIR/.git" ]]; then + cd "$PROJECT_DIR" + local branch=$(git branch --show-current 2>/dev/null || echo "unknown") + local uncommitted=$(git status --short 2>/dev/null | wc -l | tr -d ' ') + local last_commit=$(git log -1 --format="%h - %s (%cr)" 2>/dev/null || echo "No commits yet") + echo "$branch|$uncommitted|$last_commit" + else + echo "no-git|0|N/A" + fi +} + +# Check if Docker is configured +check_docker() { + if [[ -f "$PROJECT_DIR/docker-compose.yml" ]]; then + echo "yes" + else + echo "no" + fi +} + +# Check milestone progress +check_milestones() { + local completed=0 + local in_progress=0 + local total=6 + + if [[ -f "$DOCS_DIR/milestones/milestone-01-foundation.md" ]]; then + # Check for markers in milestone files + for i in $(seq 1 6); do + local file="$DOCS_DIR/milestones/milestone-0${i}-*.md" + if ls $file 1> /dev/null 2>&1; then + if grep -q "COMPLETED" $file 2>/dev/null; then + completed=$((completed + 1)) + elif grep -q "IN_PROGRESS" $file 2>/dev/null; then + in_progress=$((in_progress + 1)) + fi + fi + done + fi + + echo "$completed|$in_progress|$total" +} + +# Generate code review comments +generate_code_review() { + log "Scanning for code review items..." + + local review="" + + # Check for TODO/FIXME/XXX in code + if find "$PROJECT_DIR" -name '*.go' -o -name '*.ts' -o -name '*.tsx' | grep -v node_modules | xargs grep -n 'TODO\|FIXME\|XXX\|HACK' 2>/dev/null | head -20 > /tmp/todos.txt; then + local todo_count=$(wc -l < /tmp/todos.txt | tr -d ' ') + if [[ $todo_count -gt 0 ]]; then + review+="### Action Items Found ($todo_count)\n\n" + review+="| File | Line | Comment |\n" + review+="|------|------|---------|\n" + while IFS=: read -r file line comment; do + # Extract just the filename + local fname=$(basename "$file") + review+="| $fname | $line | $comment |\n" + done < /tmp/todos.txt + review+="\n" + fi + fi + + # Check for security issues + if find "$PROJECT_DIR" -name '*.go' | xargs grep -l 'http.Get\|http.Post\|ioutil.ReadFile' 2>/dev/null | head -5 > /tmp/security.txt; then + local sec_count=$(wc -l < /tmp/security.txt | tr -d ' ') + if [[ $sec_count -gt 0 ]]; then + review+="### Security Review Notes\n\n" + review+="- Found $sec_count files using potentially unsafe operations\n" + review+="- Ensure all HTTP requests have timeouts\n" + review+="- Validate all file paths before reading\n\n" + fi + fi + + # Check for error handling + if find "$PROJECT_DIR" -name '*.go' | xargs grep -n '_ = ' 2>/dev/null | head -10 > /tmp/errors.txt; then + local err_count=$(wc -l < /tmp/errors.txt | tr -d ' ') + if [[ $err_count -gt 0 ]]; then + review+="### Error Handling ($err_count ignored errors)\n\n" + review+="Several errors are being silently ignored. Consider handling or logging these.\n\n" + fi + fi + + echo -e "$review" +} + +# Generate the status report +generate_status() { + local iteration=$1 + local now=$(date '+%Y-%m-%d %H:%M:%S UTC') + local elapsed=$(( $(date +%s) - START_TIME )) + local elapsed_str="$((elapsed / 3600))h $(((elapsed % 3600) / 60))m" + + # Gather metrics + IFS='|' read -r go_files go_lines ts_files ts_lines md_files md_lines total_files docker_files <<< "$(analyze_codebase)" + IFS='|' read -r git_branch uncommitted last_commit <<< "$(check_git_status)" + IFS='|' read -r completed in_progress total_milestones <<< "$(check_milestones)" + local has_docker=$(check_docker) + + # Generate code review + local review=$(generate_code_review) + + # Calculate progress percentage + local progress=$((completed * 100 / total_milestones)) + + cat > "$STATUS_FILE" << EOF +# Project Status Report + +**Generated:** $now +**Monitoring Duration:** $elapsed_str +**Iteration:** $iteration +**Branch:** $git_branch + +## Quick Stats + +| Metric | Value | +|--------|-------| +| Total Files | $total_files | +| Go Files | $go_files (${go_lines} lines) | +| TypeScript Files | $ts_files (${ts_lines} lines) | +| Markdown Files | $md_files (${md_lines} lines) | +| Docker Config | $has_docker | +| Uncommitted Changes | $uncommitted | +| Milestones Complete | $completed/$total_milestones (${progress}%) | + +## Recent Activity + +**Last Commit:** $last_commit + +## Milestone Progress + +- [ ] Milestone 1: Foundation +- [ ] Milestone 2: Sync Protocol +- [ ] Milestone 3: Authentication +- [ ] Milestone 4: Collaboration +- [ ] Milestone 5: Search & UI +- [ ] Milestone 6: Production + +**Current Focus:** $([[ $in_progress -gt 0 ]] && echo "In Progress" || echo "Not Started") + +$review + +## Notes for Human Review + +EOF + + # Add contextual notes based on what we see + if [[ $go_files -eq 0 ]]; then + echo ">> **No Go files found.** Codex may be working on other aspects or hasn't started yet." >> "$STATUS_FILE" + elif [[ $go_files -lt 5 ]]; then + echo ">> **Early stage.** Only $go_files Go files exist. Foundation work likely in progress." >> "$STATUS_FILE" + else + echo ">> **Active development.** $go_files Go files with ${go_lines} lines of code." >> "$STATUS_FILE" + fi + + if [[ $uncommitted -gt 0 ]]; then + echo ">> **$uncommitted uncommitted changes** detected. Codex may be mid-work." >> "$STATUS_FILE" + fi + + if [[ -n "$review" ]]; then + echo ">> **Code review items found.** See Action Items above." >> "$STATUS_FILE" + fi + + cat >> "$STATUS_FILE" << EOF + +## Files Changed This Session + +EOF + + # List recently modified files + if [[ -f "$HASH_FILE" ]]; then + find "$PROJECT_DIR" \ + -type f \ + -mtime -0.007 \ + -not -path '*/node_modules/*' \ + -not -path '*/.git/*' \ + -not -path '*/data/*' \ + -not -name '.last_state_hash' \ + -not -name 'STATUS.md' \ + | sort | head -20 \ + | sed 's|^| - |' >> "$STATUS_FILE" || true + fi + + cat >> "$STATUS_FILE" << EOF + +--- + +*This report auto-generated by MD Hub Secure progress monitor.* +*Next update in ~10 minutes or when filesystem changes detected.* +EOF + + success "Status report updated: $STATUS_FILE" +} + +# Main monitoring loop +main() { + log "Starting MD Hub Secure progress monitor..." + log "Project: $PROJECT_DIR" + log "Watching for filesystem changes every 10 minutes" + log "Press Ctrl+C to stop, or wait for auto-exit on inactivity" + + # Ensure docs directory exists + mkdir -p "$DOCS_DIR" + + while true; do + ITERATION=$((ITERATION + 1)) + + if [[ $ITERATION -gt $MAX_ITERATIONS ]]; then + warn "Maximum iterations ($MAX_ITERATIONS) reached. Stopping." + exit 0 + fi + + log "Iteration $ITERATION - checking for changes..." + + if [[ $ITERATION -gt 1 ]] && ! has_changes; then + warn "No filesystem changes detected since last check." + warn "Stopping monitor (Codex may have finished or is inactive)." + + # Add final note to status + cat >> "$STATUS_FILE" << EOF + +## Monitor Stopped + +**Stopped:** $(date '+%Y-%m-%d %H:%M:%S UTC') +**Reason:** No filesystem activity detected + +No changes were observed between this check and the previous one. +This could mean: +- Codex has completed work +- Codex is paused or idle +- No files have been modified recently + +To restart monitoring, run: `./scripts/monitor.sh` +EOF + exit 0 + fi + + generate_status $ITERATION + + success "Iteration $ITERATION complete. Sleeping for 10 minutes..." + sleep 600 # 10 minutes + done +} + +# Handle signals +trap 'error "Monitor interrupted"; exit 1' INT TERM + +# Run main loop +main diff --git a/scripts/status.sh b/scripts/status.sh new file mode 100755 index 0000000..4dc24ab --- /dev/null +++ b/scripts/status.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# MD Hub Secure - Quick status check + +STATUS_FILE="/Users/tim/developer/md-hub-secure/docs/STATUS.md" +PID_FILE="/Users/tim/developer/md-hub-secure/.monitor.pid" + +echo "" +if [[ -f "$PID_FILE" ]]; then + PID=$(cat "$PID_FILE") + if kill -0 "$PID" 2>/dev/null; then + echo "Monitor: RUNNING (PID: $PID)" + else + echo "Monitor: STOPPED (stale PID file)" + fi +else + echo "Monitor: NOT RUNNING" +fi + +echo "" + +if [[ -f "$STATUS_FILE" ]]; then + cat "$STATUS_FILE" +else + echo "No status file found. Run ./scripts/install-monitor.sh to start monitoring." +fi