dwindown/dewemoji
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add live SQL import for local and staging sync

Browse Source
This commit is contained in:
Dwindi Ramadhana
2026-02-07 13:37:02 +07:00
parent 844ad4901b
commit 3c8f061819
5 changed files with 652 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
app/app/Services/LiveSqlImportService.php Normal file
View File

@@ -0,0 +1,328 @@
<?php
namespace App\Services;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Str;
use Symfony\Component\Console\Output\OutputInterface;
class LiveSqlImportService
{
private array $tableRename = [
'users' => 'legacy_users',
'sessions' => 'legacy_sessions',
];
private array $tableColumnMap = [
'licenses' => [
'license_key' => 'license_key',
'source' => 'source',
'plan' => 'plan',
'product_id' => 'product_id',
'status' => 'status',
'expires_at' => 'expires_at',
'meta_json' => 'meta_json',
'last_verified_at' => 'last_verified_at',
'created_at' => 'created_at',
'updated_at' => 'updated_at',
],
'license_activations' => [
'license_key' => 'license_key',
'user_id' => 'user_id',
'product' => 'product',
'device_id' => 'device_id',
'status' => 'status',
'created_at' => 'created_at',
'updated_at' => 'updated_at',
],
'usage_logs' => [
'bucket_id' => 'bucket_id',
'date_key' => 'date_key',
'used' => 'used',
'limit_count' => 'limit_count',
'seen_signatures' => 'seen_signatures',
'created_at' => 'created_at',
'updated_at' => 'updated_at',
],
];
public function import(string $path, bool $truncate, int $batchSize, ?OutputInterface $output = null): void
{
if (!is_file($path)) {
throw new \RuntimeException("SQL file not found: {$path}");
}
DB::disableQueryLog();
if ($truncate) {
$this->truncateTargets($output);
}
$file = new \SplFileObject($path, 'r');
$statement = '';
$totalStatements = 0;
$totalRows = 0;
while (!$file->eof()) {
$line = $file->fgets();
if ($line === false) {
break;
}
if ($statement === '') {
if (!Str::startsWith(ltrim($line), 'INSERT INTO')) {
continue;
}
}
$statement .= $line;
if (str_contains($line, ";\n") || str_ends_with(rtrim($line), ';')) {
$totalStatements++;
[$rowsInserted, $table] = $this->handleInsertStatement($statement, $batchSize);
$totalRows += $rowsInserted;
if ($output) {
$output->writeln("Imported {$rowsInserted} rows into {$table}");
}
$statement = '';
}
}
if ($output) {
$output->writeln("Done. Statements: {$totalStatements}. Rows: {$totalRows}.");
}
}
private function truncateTargets(?OutputInterface $output): void
{
$targets = [
'ai_guard_logs',
'ai_judgments',
'ai_lang_cache',
'ai_provider_usage',
'contributor_rewards',
'email_queue',
'emojis',
'emoji_aliases',
'emoji_keywords',
'emoji_shortcodes',
'keyword_votes',
'license_bindings',
'magic_links',
'moderation_events',
'user_prefs',
'legacy_users',
'legacy_sessions',
'licenses',
'license_activations',
'usage_logs',
];
foreach ($targets as $table) {
if (!Schema::hasTable($table)) {
continue;
}
DB::table($table)->truncate();
if ($output) {
$output->writeln("Truncated {$table}");
}
}
}
private function handleInsertStatement(string $statement, int $batchSize): array
{
$statement = trim($statement);
$pattern = '/^INSERT INTO `([^`]+)` \\(([^)]+)\\) VALUES\\s*(.+);$/s';
if (!preg_match($pattern, $statement, $matches)) {
return [0, 'unknown'];
}
$table = $matches[1];
$columnsRaw = $matches[2];
$valuesRaw = $matches[3];
$columns = array_map(
static fn (string $col): string => trim($col, " `"),
explode(',', $columnsRaw)
);
$targetTable = $this->tableRename[$table] ?? $table;
if (!Schema::hasTable($targetTable)) {
return [0, $targetTable];
}
$rows = $this->parseValues($valuesRaw, count($columns));
$total = 0;
$batch = [];
foreach ($rows as $rowValues) {
$row = array_combine($columns, $rowValues);
$mapped = $this->mapRow($table, $row);
if ($mapped === null) {
continue;
}
$batch[] = $mapped;
if (count($batch) >= $batchSize) {
DB::table($targetTable)->insert($batch);
$total += count($batch);
$batch = [];
}
}
if ($batch !== []) {
DB::table($targetTable)->insert($batch);
$total += count($batch);
}
return [$total, $targetTable];
}
private function mapRow(string $table, array $row): ?array
{
if (isset($this->tableColumnMap[$table])) {
$mapped = [];
foreach ($this->tableColumnMap[$table] as $from => $to) {
$mapped[$to] = $row[$from] ?? null;
}
if ($table === 'licenses') {
$mapped['owner_user_id'] = null;
}
return $mapped;
}
if ($table === 'users') {
return [
'user_id' => $row['user_id'] ?? null,
'email' => $row['email'] ?? null,
'username' => $row['username'] ?? null,
'opted_in' => $row['opted_in'] ?? 0,
'created_at' => $row['created_at'] ?? null,
'updated_at' => $row['updated_at'] ?? null,
];
}
if ($table === 'sessions') {
return [
'session_id' => $row['session_id'] ?? null,
'user_id' => $row['user_id'] ?? null,
'ip_hash' => $row['ip_hash'] ?? null,
'ua_hash' => $row['ua_hash'] ?? null,
'expires_at' => $row['expires_at'] ?? null,
'created_at' => $row['created_at'] ?? null,
];
}
return $row;
}
private function parseValues(string $valuesRaw, int $columnCount): array
{
$rows = [];
$currentRow = [];
$buffer = '';
$inString = false;
$escape = false;
$valueIsString = false;
$inRow = false;
$length = strlen($valuesRaw);
for ($i = 0; $i < $length; $i++) {
$ch = $valuesRaw[$i];
if ($inString) {
if ($escape) {
$buffer .= $this->unescapeChar($ch);
$escape = false;
continue;
}
if ($ch === '\\\\') {
$escape = true;
continue;
}
if ($ch === '\'') {
$inString = false;
continue;
}
$buffer .= $ch;
continue;
}
if ($ch === '(') {
$inRow = true;
$currentRow = [];
$buffer = '';
$valueIsString = false;
continue;
}
if ($ch === '\'') {
$inString = true;
$valueIsString = true;
continue;
}
if ($ch === ',' && $inRow) {
$currentRow[] = $this->convertValue($buffer, $valueIsString);
$buffer = '';
$valueIsString = false;
continue;
}
if ($ch === ')' && $inRow) {
$currentRow[] = $this->convertValue($buffer, $valueIsString);
if (count($currentRow) === $columnCount) {
$rows[] = $currentRow;
}
$buffer = '';
$valueIsString = false;
$inRow = false;
continue;
}
if ($inRow) {
$buffer .= $ch;
}
}
return $rows;
}
private function convertValue(string $buffer, bool $valueIsString): mixed
{
$value = trim($buffer);
if (!$valueIsString) {
if ($value === '' || strtoupper($value) === 'NULL') {
return null;
}
if (is_numeric($value)) {
return $value + 0;
}
}
return $value;
}
private function unescapeChar(string $ch): string
{
return match ($ch) {
'n' => "\n",
'r' => "\r",
't' => "\t",
'0' => "\0",
'Z' => "\x1A",
default => $ch,
};
}
}

202
app/database/migrations/2026_02_06_000000_create_live_import_tables.php Normal file
View File

@@ -0,0 +1,202 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::create('ai_guard_logs', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->string('emoji_slug', 64);
$table->string('lang', 8);
$table->string('region', 8)->nullable();
$table->string('keyword', 128);
$table->string('decision', 16);
$table->string('reason', 32);
$table->string('detected_lang', 8)->nullable();
$table->string('detail_en', 255)->nullable();
$table->string('detail_local', 255)->nullable();
$table->timestamp('created_at')->nullable();
});
Schema::create('ai_judgments', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->unsignedBigInteger('keyword_id')->nullable();
$table->string('emoji_slug', 190);
$table->string('lang', 16);
$table->string('keyword', 64);
$table->string('model', 64)->default('gemini-free');
$table->string('verdict', 16);
$table->decimal('confidence', 5, 4)->nullable();
$table->longText('raw_response')->nullable();
$table->timestamp('created_at')->nullable();
});
Schema::create('ai_lang_cache', function (Blueprint $table): void {
$table->string('keyword', 128)->primary();
$table->string('detected_lang', 8);
$table->timestamp('last_seen')->nullable();
});
Schema::create('ai_provider_usage', function (Blueprint $table): void {
$table->string('provider', 96);
$table->string('period', 7);
$table->unsignedBigInteger('tokens_used')->default(0);
$table->primary(['provider', 'period']);
});
Schema::create('contributor_rewards', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->string('user_id', 64);
$table->string('type', 32);
$table->string('reason', 128);
$table->string('issued_by', 64)->default('system');
$table->timestamp('issued_at')->nullable();
$table->timestamp('redeemed_at')->nullable();
$table->longText('meta')->nullable();
});
Schema::create('email_queue', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->string('user_id', 64)->nullable();
$table->string('to_email', 255);
$table->string('template', 64);
$table->longText('payload')->nullable();
$table->string('provider', 32)->default('elastic_email');
$table->string('status', 32)->default('queued');
$table->string('last_error', 255)->nullable();
$table->timestamp('created_at')->nullable();
$table->timestamp('sent_at')->nullable();
});
Schema::create('emojis', function (Blueprint $table): void {
$table->unsignedInteger('emoji_id')->primary();
$table->string('slug', 191);
$table->string('emoji_char', 64);
$table->string('name', 191);
$table->string('category', 128);
$table->string('subcategory', 128);
$table->string('unified', 128);
$table->string('default_presentation', 16);
$table->string('version', 16);
$table->boolean('supports_skin_tone')->default(false);
$table->string('permalink', 255);
$table->text('description')->nullable();
$table->string('meta_title', 255)->nullable();
$table->string('meta_description', 255)->nullable();
$table->string('title', 255)->nullable();
$table->timestamp('created_at')->nullable();
$table->timestamp('updated_at')->nullable();
});
Schema::create('emoji_aliases', function (Blueprint $table): void {
$table->unsignedInteger('emoji_id');
$table->string('alias', 191);
$table->unique(['emoji_id', 'alias']);
});
Schema::create('emoji_keywords', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->string('emoji_slug', 190);
$table->string('lang', 2);
$table->string('region', 2)->default('');
$table->string('keyword', 64);
$table->string('status', 16)->default('private');
$table->string('owner_user_id', 64)->nullable();
$table->string('visibility', 32)->default('private');
$table->string('public_key', 900)->nullable();
$table->timestamp('created_at')->nullable();
$table->timestamp('updated_at')->nullable();
});
Schema::create('emoji_shortcodes', function (Blueprint $table): void {
$table->unsignedInteger('emoji_id');
$table->string('shortcode', 191);
$table->string('kind', 16)->default('primary');
$table->unique(['emoji_id', 'shortcode', 'kind']);
});
Schema::create('keyword_votes', function (Blueprint $table): void {
$table->unsignedBigInteger('keyword_id');
$table->string('user_id', 64);
$table->string('vote', 8);
$table->tinyInteger('value');
$table->timestamp('created_at')->nullable();
$table->primary(['keyword_id', 'user_id']);
});
Schema::create('license_bindings', function (Blueprint $table): void {
$table->string('license_key', 64);
$table->string('user_id', 64);
$table->timestamp('created_at')->nullable();
$table->unique(['license_key', 'user_id']);
});
Schema::create('magic_links', function (Blueprint $table): void {
$table->string('token_hash', 64)->primary();
$table->string('user_id', 64);
$table->string('purpose', 32)->default('login');
$table->timestamp('expires_at');
$table->timestamp('used_at')->nullable();
$table->timestamp('created_at')->nullable();
});
Schema::create('moderation_events', function (Blueprint $table): void {
$table->bigIncrements('id');
$table->unsignedBigInteger('keyword_id');
$table->string('old_status', 16)->nullable();
$table->string('new_status', 16);
$table->string('reason', 255)->nullable();
$table->string('actor', 64)->default('system');
$table->timestamp('created_at')->nullable();
});
Schema::create('user_prefs', function (Blueprint $table): void {
$table->string('user_id', 64)->primary();
$table->longText('preferred_languages');
$table->timestamp('updated_at')->nullable();
});
Schema::create('legacy_users', function (Blueprint $table): void {
$table->string('user_id', 64)->primary();
$table->string('email', 255)->nullable();
$table->string('username', 32);
$table->boolean('opted_in')->default(false);
$table->timestamp('created_at')->nullable();
$table->timestamp('updated_at')->nullable();
});
Schema::create('legacy_sessions', function (Blueprint $table): void {
$table->string('session_id', 64)->primary();
$table->string('user_id', 64);
$table->string('ip_hash', 64)->nullable();
$table->string('ua_hash', 64)->nullable();
$table->timestamp('expires_at');
$table->timestamp('created_at')->nullable();
});
}
public function down(): void
{
Schema::dropIfExists('legacy_sessions');
Schema::dropIfExists('legacy_users');
Schema::dropIfExists('user_prefs');
Schema::dropIfExists('moderation_events');
Schema::dropIfExists('magic_links');
Schema::dropIfExists('license_bindings');
Schema::dropIfExists('keyword_votes');
Schema::dropIfExists('emoji_shortcodes');
Schema::dropIfExists('emoji_keywords');
Schema::dropIfExists('emoji_aliases');
Schema::dropIfExists('emojis');
Schema::dropIfExists('email_queue');
Schema::dropIfExists('contributor_rewards');
Schema::dropIfExists('ai_provider_usage');
Schema::dropIfExists('ai_lang_cache');
Schema::dropIfExists('ai_judgments');
Schema::dropIfExists('ai_guard_logs');
}
};

10
app/routes/console.php
View File

@@ -2,7 +2,17 @@
use Illuminate\Foundation\Inspiring;
use Illuminate\Support\Facades\Artisan;
use App\Services\LiveSqlImportService;
Artisan::command('inspire', function () {
$this->comment(Inspiring::quote());
})->purpose('Display an inspiring quote');
Artisan::command('dewemoji:import-live-sql {path : Absolute path to dewemojiAPI_DB.sql} {--truncate : Truncate target tables first} {--batch=500 : Insert batch size}', function () {
$path = (string) $this->argument('path');
$truncate = (bool) $this->option('truncate');
$batch = (int) $this->option('batch');
$importer = app(LiveSqlImportService::class);
$importer->import($path, $truncate, $batch, $this->output);
})->purpose('Import live SQL dump into the current database');