diff --git a/app/app/Services/LiveSqlImportService.php b/app/app/Services/LiveSqlImportService.php new file mode 100644 index 0000000..7884066 --- /dev/null +++ b/app/app/Services/LiveSqlImportService.php @@ -0,0 +1,328 @@ + 'legacy_users', + 'sessions' => 'legacy_sessions', + ]; + + private array $tableColumnMap = [ + 'licenses' => [ + 'license_key' => 'license_key', + 'source' => 'source', + 'plan' => 'plan', + 'product_id' => 'product_id', + 'status' => 'status', + 'expires_at' => 'expires_at', + 'meta_json' => 'meta_json', + 'last_verified_at' => 'last_verified_at', + 'created_at' => 'created_at', + 'updated_at' => 'updated_at', + ], + 'license_activations' => [ + 'license_key' => 'license_key', + 'user_id' => 'user_id', + 'product' => 'product', + 'device_id' => 'device_id', + 'status' => 'status', + 'created_at' => 'created_at', + 'updated_at' => 'updated_at', + ], + 'usage_logs' => [ + 'bucket_id' => 'bucket_id', + 'date_key' => 'date_key', + 'used' => 'used', + 'limit_count' => 'limit_count', + 'seen_signatures' => 'seen_signatures', + 'created_at' => 'created_at', + 'updated_at' => 'updated_at', + ], + ]; + + public function import(string $path, bool $truncate, int $batchSize, ?OutputInterface $output = null): void + { + if (!is_file($path)) { + throw new \RuntimeException("SQL file not found: {$path}"); + } + + DB::disableQueryLog(); + + if ($truncate) { + $this->truncateTargets($output); + } + + $file = new \SplFileObject($path, 'r'); + $statement = ''; + $totalStatements = 0; + $totalRows = 0; + + while (!$file->eof()) { + $line = $file->fgets(); + if ($line === false) { + break; + } + + if ($statement === '') { + if (!Str::startsWith(ltrim($line), 'INSERT INTO')) { + continue; + } + } + + $statement .= $line; + if (str_contains($line, ";\n") || str_ends_with(rtrim($line), ';')) { + $totalStatements++; + [$rowsInserted, $table] = $this->handleInsertStatement($statement, $batchSize); + $totalRows += $rowsInserted; + + if ($output) { + $output->writeln("Imported {$rowsInserted} rows into {$table}"); + } + + $statement = ''; + } + } + + if ($output) { + $output->writeln("Done. Statements: {$totalStatements}. Rows: {$totalRows}."); + } + } + + private function truncateTargets(?OutputInterface $output): void + { + $targets = [ + 'ai_guard_logs', + 'ai_judgments', + 'ai_lang_cache', + 'ai_provider_usage', + 'contributor_rewards', + 'email_queue', + 'emojis', + 'emoji_aliases', + 'emoji_keywords', + 'emoji_shortcodes', + 'keyword_votes', + 'license_bindings', + 'magic_links', + 'moderation_events', + 'user_prefs', + 'legacy_users', + 'legacy_sessions', + 'licenses', + 'license_activations', + 'usage_logs', + ]; + + foreach ($targets as $table) { + if (!Schema::hasTable($table)) { + continue; + } + DB::table($table)->truncate(); + if ($output) { + $output->writeln("Truncated {$table}"); + } + } + } + + private function handleInsertStatement(string $statement, int $batchSize): array + { + $statement = trim($statement); + $pattern = '/^INSERT INTO `([^`]+)` \\(([^)]+)\\) VALUES\\s*(.+);$/s'; + + if (!preg_match($pattern, $statement, $matches)) { + return [0, 'unknown']; + } + + $table = $matches[1]; + $columnsRaw = $matches[2]; + $valuesRaw = $matches[3]; + + $columns = array_map( + static fn (string $col): string => trim($col, " `"), + explode(',', $columnsRaw) + ); + + $targetTable = $this->tableRename[$table] ?? $table; + + if (!Schema::hasTable($targetTable)) { + return [0, $targetTable]; + } + + $rows = $this->parseValues($valuesRaw, count($columns)); + $total = 0; + $batch = []; + + foreach ($rows as $rowValues) { + $row = array_combine($columns, $rowValues); + $mapped = $this->mapRow($table, $row); + if ($mapped === null) { + continue; + } + + $batch[] = $mapped; + if (count($batch) >= $batchSize) { + DB::table($targetTable)->insert($batch); + $total += count($batch); + $batch = []; + } + } + + if ($batch !== []) { + DB::table($targetTable)->insert($batch); + $total += count($batch); + } + + return [$total, $targetTable]; + } + + private function mapRow(string $table, array $row): ?array + { + if (isset($this->tableColumnMap[$table])) { + $mapped = []; + foreach ($this->tableColumnMap[$table] as $from => $to) { + $mapped[$to] = $row[$from] ?? null; + } + + if ($table === 'licenses') { + $mapped['owner_user_id'] = null; + } + + return $mapped; + } + + if ($table === 'users') { + return [ + 'user_id' => $row['user_id'] ?? null, + 'email' => $row['email'] ?? null, + 'username' => $row['username'] ?? null, + 'opted_in' => $row['opted_in'] ?? 0, + 'created_at' => $row['created_at'] ?? null, + 'updated_at' => $row['updated_at'] ?? null, + ]; + } + + if ($table === 'sessions') { + return [ + 'session_id' => $row['session_id'] ?? null, + 'user_id' => $row['user_id'] ?? null, + 'ip_hash' => $row['ip_hash'] ?? null, + 'ua_hash' => $row['ua_hash'] ?? null, + 'expires_at' => $row['expires_at'] ?? null, + 'created_at' => $row['created_at'] ?? null, + ]; + } + + return $row; + } + + private function parseValues(string $valuesRaw, int $columnCount): array + { + $rows = []; + $currentRow = []; + $buffer = ''; + $inString = false; + $escape = false; + $valueIsString = false; + $inRow = false; + $length = strlen($valuesRaw); + + for ($i = 0; $i < $length; $i++) { + $ch = $valuesRaw[$i]; + + if ($inString) { + if ($escape) { + $buffer .= $this->unescapeChar($ch); + $escape = false; + continue; + } + + if ($ch === '\\\\') { + $escape = true; + continue; + } + + if ($ch === '\'') { + $inString = false; + continue; + } + + $buffer .= $ch; + continue; + } + + if ($ch === '(') { + $inRow = true; + $currentRow = []; + $buffer = ''; + $valueIsString = false; + continue; + } + + if ($ch === '\'') { + $inString = true; + $valueIsString = true; + continue; + } + + if ($ch === ',' && $inRow) { + $currentRow[] = $this->convertValue($buffer, $valueIsString); + $buffer = ''; + $valueIsString = false; + continue; + } + + if ($ch === ')' && $inRow) { + $currentRow[] = $this->convertValue($buffer, $valueIsString); + if (count($currentRow) === $columnCount) { + $rows[] = $currentRow; + } + $buffer = ''; + $valueIsString = false; + $inRow = false; + continue; + } + + if ($inRow) { + $buffer .= $ch; + } + } + + return $rows; + } + + private function convertValue(string $buffer, bool $valueIsString): mixed + { + $value = trim($buffer); + + if (!$valueIsString) { + if ($value === '' || strtoupper($value) === 'NULL') { + return null; + } + + if (is_numeric($value)) { + return $value + 0; + } + } + + return $value; + } + + private function unescapeChar(string $ch): string + { + return match ($ch) { + 'n' => "\n", + 'r' => "\r", + 't' => "\t", + '0' => "\0", + 'Z' => "\x1A", + default => $ch, + }; + } +} diff --git a/app/database/migrations/2026_02_06_000000_create_live_import_tables.php b/app/database/migrations/2026_02_06_000000_create_live_import_tables.php new file mode 100644 index 0000000..6a02b6a --- /dev/null +++ b/app/database/migrations/2026_02_06_000000_create_live_import_tables.php @@ -0,0 +1,202 @@ +bigIncrements('id'); + $table->string('emoji_slug', 64); + $table->string('lang', 8); + $table->string('region', 8)->nullable(); + $table->string('keyword', 128); + $table->string('decision', 16); + $table->string('reason', 32); + $table->string('detected_lang', 8)->nullable(); + $table->string('detail_en', 255)->nullable(); + $table->string('detail_local', 255)->nullable(); + $table->timestamp('created_at')->nullable(); + }); + + Schema::create('ai_judgments', function (Blueprint $table): void { + $table->bigIncrements('id'); + $table->unsignedBigInteger('keyword_id')->nullable(); + $table->string('emoji_slug', 190); + $table->string('lang', 16); + $table->string('keyword', 64); + $table->string('model', 64)->default('gemini-free'); + $table->string('verdict', 16); + $table->decimal('confidence', 5, 4)->nullable(); + $table->longText('raw_response')->nullable(); + $table->timestamp('created_at')->nullable(); + }); + + Schema::create('ai_lang_cache', function (Blueprint $table): void { + $table->string('keyword', 128)->primary(); + $table->string('detected_lang', 8); + $table->timestamp('last_seen')->nullable(); + }); + + Schema::create('ai_provider_usage', function (Blueprint $table): void { + $table->string('provider', 96); + $table->string('period', 7); + $table->unsignedBigInteger('tokens_used')->default(0); + $table->primary(['provider', 'period']); + }); + + Schema::create('contributor_rewards', function (Blueprint $table): void { + $table->bigIncrements('id'); + $table->string('user_id', 64); + $table->string('type', 32); + $table->string('reason', 128); + $table->string('issued_by', 64)->default('system'); + $table->timestamp('issued_at')->nullable(); + $table->timestamp('redeemed_at')->nullable(); + $table->longText('meta')->nullable(); + }); + + Schema::create('email_queue', function (Blueprint $table): void { + $table->bigIncrements('id'); + $table->string('user_id', 64)->nullable(); + $table->string('to_email', 255); + $table->string('template', 64); + $table->longText('payload')->nullable(); + $table->string('provider', 32)->default('elastic_email'); + $table->string('status', 32)->default('queued'); + $table->string('last_error', 255)->nullable(); + $table->timestamp('created_at')->nullable(); + $table->timestamp('sent_at')->nullable(); + }); + + Schema::create('emojis', function (Blueprint $table): void { + $table->unsignedInteger('emoji_id')->primary(); + $table->string('slug', 191); + $table->string('emoji_char', 64); + $table->string('name', 191); + $table->string('category', 128); + $table->string('subcategory', 128); + $table->string('unified', 128); + $table->string('default_presentation', 16); + $table->string('version', 16); + $table->boolean('supports_skin_tone')->default(false); + $table->string('permalink', 255); + $table->text('description')->nullable(); + $table->string('meta_title', 255)->nullable(); + $table->string('meta_description', 255)->nullable(); + $table->string('title', 255)->nullable(); + $table->timestamp('created_at')->nullable(); + $table->timestamp('updated_at')->nullable(); + }); + + Schema::create('emoji_aliases', function (Blueprint $table): void { + $table->unsignedInteger('emoji_id'); + $table->string('alias', 191); + $table->unique(['emoji_id', 'alias']); + }); + + Schema::create('emoji_keywords', function (Blueprint $table): void { + $table->bigIncrements('id'); + $table->string('emoji_slug', 190); + $table->string('lang', 2); + $table->string('region', 2)->default(''); + $table->string('keyword', 64); + $table->string('status', 16)->default('private'); + $table->string('owner_user_id', 64)->nullable(); + $table->string('visibility', 32)->default('private'); + $table->string('public_key', 900)->nullable(); + $table->timestamp('created_at')->nullable(); + $table->timestamp('updated_at')->nullable(); + }); + + Schema::create('emoji_shortcodes', function (Blueprint $table): void { + $table->unsignedInteger('emoji_id'); + $table->string('shortcode', 191); + $table->string('kind', 16)->default('primary'); + $table->unique(['emoji_id', 'shortcode', 'kind']); + }); + + Schema::create('keyword_votes', function (Blueprint $table): void { + $table->unsignedBigInteger('keyword_id'); + $table->string('user_id', 64); + $table->string('vote', 8); + $table->tinyInteger('value'); + $table->timestamp('created_at')->nullable(); + $table->primary(['keyword_id', 'user_id']); + }); + + Schema::create('license_bindings', function (Blueprint $table): void { + $table->string('license_key', 64); + $table->string('user_id', 64); + $table->timestamp('created_at')->nullable(); + $table->unique(['license_key', 'user_id']); + }); + + Schema::create('magic_links', function (Blueprint $table): void { + $table->string('token_hash', 64)->primary(); + $table->string('user_id', 64); + $table->string('purpose', 32)->default('login'); + $table->timestamp('expires_at'); + $table->timestamp('used_at')->nullable(); + $table->timestamp('created_at')->nullable(); + }); + + Schema::create('moderation_events', function (Blueprint $table): void { + $table->bigIncrements('id'); + $table->unsignedBigInteger('keyword_id'); + $table->string('old_status', 16)->nullable(); + $table->string('new_status', 16); + $table->string('reason', 255)->nullable(); + $table->string('actor', 64)->default('system'); + $table->timestamp('created_at')->nullable(); + }); + + Schema::create('user_prefs', function (Blueprint $table): void { + $table->string('user_id', 64)->primary(); + $table->longText('preferred_languages'); + $table->timestamp('updated_at')->nullable(); + }); + + Schema::create('legacy_users', function (Blueprint $table): void { + $table->string('user_id', 64)->primary(); + $table->string('email', 255)->nullable(); + $table->string('username', 32); + $table->boolean('opted_in')->default(false); + $table->timestamp('created_at')->nullable(); + $table->timestamp('updated_at')->nullable(); + }); + + Schema::create('legacy_sessions', function (Blueprint $table): void { + $table->string('session_id', 64)->primary(); + $table->string('user_id', 64); + $table->string('ip_hash', 64)->nullable(); + $table->string('ua_hash', 64)->nullable(); + $table->timestamp('expires_at'); + $table->timestamp('created_at')->nullable(); + }); + } + + public function down(): void + { + Schema::dropIfExists('legacy_sessions'); + Schema::dropIfExists('legacy_users'); + Schema::dropIfExists('user_prefs'); + Schema::dropIfExists('moderation_events'); + Schema::dropIfExists('magic_links'); + Schema::dropIfExists('license_bindings'); + Schema::dropIfExists('keyword_votes'); + Schema::dropIfExists('emoji_shortcodes'); + Schema::dropIfExists('emoji_keywords'); + Schema::dropIfExists('emoji_aliases'); + Schema::dropIfExists('emojis'); + Schema::dropIfExists('email_queue'); + Schema::dropIfExists('contributor_rewards'); + Schema::dropIfExists('ai_provider_usage'); + Schema::dropIfExists('ai_lang_cache'); + Schema::dropIfExists('ai_judgments'); + Schema::dropIfExists('ai_guard_logs'); + } +}; diff --git a/app/routes/console.php b/app/routes/console.php index 3c9adf1..71c153c 100644 --- a/app/routes/console.php +++ b/app/routes/console.php @@ -2,7 +2,17 @@ use Illuminate\Foundation\Inspiring; use Illuminate\Support\Facades\Artisan; +use App\Services\LiveSqlImportService; Artisan::command('inspire', function () { $this->comment(Inspiring::quote()); })->purpose('Display an inspiring quote'); + +Artisan::command('dewemoji:import-live-sql {path : Absolute path to dewemojiAPI_DB.sql} {--truncate : Truncate target tables first} {--batch=500 : Insert batch size}', function () { + $path = (string) $this->argument('path'); + $truncate = (bool) $this->option('truncate'); + $batch = (int) $this->option('batch'); + + $importer = app(LiveSqlImportService::class); + $importer->import($path, $truncate, $batch, $this->output); +})->purpose('Import live SQL dump into the current database'); diff --git a/current-local-db.md b/current-local-db.md new file mode 100644 index 0000000..2b8d510 --- /dev/null +++ b/current-local-db.md @@ -0,0 +1,74 @@ +# Current Local Database (SQLite) + +This describes what exists **right now** in the rebuild app (`app/database/database.sqlite`). + +## Local DB engine + +- SQLite (file: `app/database/database.sqlite`) + +## Tables present + +Framework defaults: +- `cache` +- `cache_locks` +- `failed_jobs` +- `job_batches` +- `jobs` +- `migrations` +- `password_reset_tokens` +- `sessions` +- `users` + +Dewemoji core tables: +- `licenses` +- `license_activations` +- `usage_logs` + +## Current row counts (local) + +- `emojis`: 2131 +- `emoji_keywords`: 13420 +- `emoji_aliases`: 0 +- `emoji_shortcodes`: 0 +- `licenses`: 7 +- `license_activations`: 1 +- `usage_logs`: 88 +- `ai_guard_logs`: 54 +- `ai_judgments`: 0 +- `ai_lang_cache`: 6 +- `ai_provider_usage`: 4 +- `legacy_users`: 0 +- `legacy_sessions`: 0 + +## What is *not* in local DB yet (from live SQL) + +From `dewemoji-live-backend/dewemojiAPI_DB.sql`, these tables exist in live but are **still empty locally**: + +- `emoji_aliases` +- `emoji_shortcodes` +- `ai_judgments` +- `legacy_users` (live users) +- `legacy_sessions` (live sessions) + +## Why emojis still work locally + +The rebuild app currently reads emojis from a **JSON dataset** (not DB): +- `app/data/emojis.json` + +That’s why the UI works even though emoji tables aren’t present yet. + +## Next step (if you want to migrate live SQL) + +Migrations + importer are now in place. To sync everything locally (SQLite), run: + +```bash +cd /Users/dwindown/Developments/dewemoji/app +php artisan migrate +php artisan dewemoji:import-live-sql /Users/dwindown/Developments/dewemoji-live-backend/dewemojiAPI_DB.sql --truncate +``` + +Notes: +- Live `users` + `sessions` go into `legacy_users` + `legacy_sessions` (so Laravel auth/session tables stay safe). +- Licenses/activations/usage_logs are mapped into the current tables for parity. + +Just tell me which subset to migrate first. diff --git a/staging-sync-checklist.md b/staging-sync-checklist.md new file mode 100644 index 0000000..bcebd29 --- /dev/null +++ b/staging-sync-checklist.md @@ -0,0 +1,38 @@ +# Staging Sync Checklist (MySQL) + +This is the exact, minimal checklist to sync **live SQL → staging MySQL**. + +## 1) Point app to MySQL + +Edit `app/.env` (or set in Coolify ENV): + +``` +DB_CONNECTION=mysql +DB_HOST=YOUR_MYSQL_HOST +DB_PORT=3306 +DB_DATABASE=YOUR_DB +DB_USERNAME=YOUR_USER +DB_PASSWORD=YOUR_PASS +``` + +## 2) Run migrations + import + +```bash +cd /Users/dwindown/Developments/dewemoji/app +php artisan migrate +php artisan dewemoji:import-live-sql /Users/dwindown/Developments/dewemoji-live-backend/dewemojiAPI_DB.sql --truncate +``` + +## 3) Quick sanity checks + +```bash +php artisan tinker --execute="echo DB::table('emojis')->count().PHP_EOL;" +php artisan tinker --execute="echo DB::table('emoji_keywords')->count().PHP_EOL;" +``` + +Expected: `emojis` ~ 2131, `emoji_keywords` ~ 13420. + +## Notes + +- Live `users` + `sessions` are imported into `legacy_users` + `legacy_sessions`. +- Licenses/activations/usage logs are imported into current tables for parity.