Add live SQL import for local and staging sync
This commit is contained in:
Dwindi Ramadhana
328
app/app/Services/LiveSqlImportService.php
Normal file
328
app/app/Services/LiveSqlImportService.php
Normal file
@@ -0,0 +1,328 @@
|
||||
<?php
|
||||
|
||||
namespace App\Services;
|
||||
|
||||
use Illuminate\Support\Facades\DB;
|
||||
use Illuminate\Support\Facades\Schema;
|
||||
use Illuminate\Support\Str;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
|
||||
class LiveSqlImportService
|
||||
{
|
||||
private array $tableRename = [
|
||||
'users' => 'legacy_users',
|
||||
'sessions' => 'legacy_sessions',
|
||||
];
|
||||
|
||||
private array $tableColumnMap = [
|
||||
'licenses' => [
|
||||
'license_key' => 'license_key',
|
||||
'source' => 'source',
|
||||
'plan' => 'plan',
|
||||
'product_id' => 'product_id',
|
||||
'status' => 'status',
|
||||
'expires_at' => 'expires_at',
|
||||
'meta_json' => 'meta_json',
|
||||
'last_verified_at' => 'last_verified_at',
|
||||
'created_at' => 'created_at',
|
||||
'updated_at' => 'updated_at',
|
||||
],
|
||||
'license_activations' => [
|
||||
'license_key' => 'license_key',
|
||||
'user_id' => 'user_id',
|
||||
'product' => 'product',
|
||||
'device_id' => 'device_id',
|
||||
'status' => 'status',
|
||||
'created_at' => 'created_at',
|
||||
'updated_at' => 'updated_at',
|
||||
],
|
||||
'usage_logs' => [
|
||||
'bucket_id' => 'bucket_id',
|
||||
'date_key' => 'date_key',
|
||||
'used' => 'used',
|
||||
'limit_count' => 'limit_count',
|
||||
'seen_signatures' => 'seen_signatures',
|
||||
'created_at' => 'created_at',
|
||||
'updated_at' => 'updated_at',
|
||||
],
|
||||
];
|
||||
|
||||
public function import(string $path, bool $truncate, int $batchSize, ?OutputInterface $output = null): void
|
||||
{
|
||||
if (!is_file($path)) {
|
||||
throw new \RuntimeException("SQL file not found: {$path}");
|
||||
}
|
||||
|
||||
DB::disableQueryLog();
|
||||
|
||||
if ($truncate) {
|
||||
$this->truncateTargets($output);
|
||||
}
|
||||
|
||||
$file = new \SplFileObject($path, 'r');
|
||||
$statement = '';
|
||||
$totalStatements = 0;
|
||||
$totalRows = 0;
|
||||
|
||||
while (!$file->eof()) {
|
||||
$line = $file->fgets();
|
||||
if ($line === false) {
|
||||
break;
|
||||
}
|
||||
|
||||
if ($statement === '') {
|
||||
if (!Str::startsWith(ltrim($line), 'INSERT INTO')) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
$statement .= $line;
|
||||
if (str_contains($line, ";\n") || str_ends_with(rtrim($line), ';')) {
|
||||
$totalStatements++;
|
||||
[$rowsInserted, $table] = $this->handleInsertStatement($statement, $batchSize);
|
||||
$totalRows += $rowsInserted;
|
||||
|
||||
if ($output) {
|
||||
$output->writeln("Imported {$rowsInserted} rows into {$table}");
|
||||
}
|
||||
|
||||
$statement = '';
|
||||
}
|
||||
}
|
||||
|
||||
if ($output) {
|
||||
$output->writeln("Done. Statements: {$totalStatements}. Rows: {$totalRows}.");
|
||||
}
|
||||
}
|
||||
|
||||
private function truncateTargets(?OutputInterface $output): void
|
||||
{
|
||||
$targets = [
|
||||
'ai_guard_logs',
|
||||
'ai_judgments',
|
||||
'ai_lang_cache',
|
||||
'ai_provider_usage',
|
||||
'contributor_rewards',
|
||||
'email_queue',
|
||||
'emojis',
|
||||
'emoji_aliases',
|
||||
'emoji_keywords',
|
||||
'emoji_shortcodes',
|
||||
'keyword_votes',
|
||||
'license_bindings',
|
||||
'magic_links',
|
||||
'moderation_events',
|
||||
'user_prefs',
|
||||
'legacy_users',
|
||||
'legacy_sessions',
|
||||
'licenses',
|
||||
'license_activations',
|
||||
'usage_logs',
|
||||
];
|
||||
|
||||
foreach ($targets as $table) {
|
||||
if (!Schema::hasTable($table)) {
|
||||
continue;
|
||||
}
|
||||
DB::table($table)->truncate();
|
||||
if ($output) {
|
||||
$output->writeln("Truncated {$table}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function handleInsertStatement(string $statement, int $batchSize): array
|
||||
{
|
||||
$statement = trim($statement);
|
||||
$pattern = '/^INSERT INTO `([^`]+)` \\(([^)]+)\\) VALUES\\s*(.+);$/s';
|
||||
|
||||
if (!preg_match($pattern, $statement, $matches)) {
|
||||
return [0, 'unknown'];
|
||||
}
|
||||
|
||||
$table = $matches[1];
|
||||
$columnsRaw = $matches[2];
|
||||
$valuesRaw = $matches[3];
|
||||
|
||||
$columns = array_map(
|
||||
static fn (string $col): string => trim($col, " `"),
|
||||
explode(',', $columnsRaw)
|
||||
);
|
||||
|
||||
$targetTable = $this->tableRename[$table] ?? $table;
|
||||
|
||||
if (!Schema::hasTable($targetTable)) {
|
||||
return [0, $targetTable];
|
||||
}
|
||||
|
||||
$rows = $this->parseValues($valuesRaw, count($columns));
|
||||
$total = 0;
|
||||
$batch = [];
|
||||
|
||||
foreach ($rows as $rowValues) {
|
||||
$row = array_combine($columns, $rowValues);
|
||||
$mapped = $this->mapRow($table, $row);
|
||||
if ($mapped === null) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$batch[] = $mapped;
|
||||
if (count($batch) >= $batchSize) {
|
||||
DB::table($targetTable)->insert($batch);
|
||||
$total += count($batch);
|
||||
$batch = [];
|
||||
}
|
||||
}
|
||||
|
||||
if ($batch !== []) {
|
||||
DB::table($targetTable)->insert($batch);
|
||||
$total += count($batch);
|
||||
}
|
||||
|
||||
return [$total, $targetTable];
|
||||
}
|
||||
|
||||
private function mapRow(string $table, array $row): ?array
|
||||
{
|
||||
if (isset($this->tableColumnMap[$table])) {
|
||||
$mapped = [];
|
||||
foreach ($this->tableColumnMap[$table] as $from => $to) {
|
||||
$mapped[$to] = $row[$from] ?? null;
|
||||
}
|
||||
|
||||
if ($table === 'licenses') {
|
||||
$mapped['owner_user_id'] = null;
|
||||
}
|
||||
|
||||
return $mapped;
|
||||
}
|
||||
|
||||
if ($table === 'users') {
|
||||
return [
|
||||
'user_id' => $row['user_id'] ?? null,
|
||||
'email' => $row['email'] ?? null,
|
||||
'username' => $row['username'] ?? null,
|
||||
'opted_in' => $row['opted_in'] ?? 0,
|
||||
'created_at' => $row['created_at'] ?? null,
|
||||
'updated_at' => $row['updated_at'] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
if ($table === 'sessions') {
|
||||
return [
|
||||
'session_id' => $row['session_id'] ?? null,
|
||||
'user_id' => $row['user_id'] ?? null,
|
||||
'ip_hash' => $row['ip_hash'] ?? null,
|
||||
'ua_hash' => $row['ua_hash'] ?? null,
|
||||
'expires_at' => $row['expires_at'] ?? null,
|
||||
'created_at' => $row['created_at'] ?? null,
|
||||
];
|
||||
}
|
||||
|
||||
return $row;
|
||||
}
|
||||
|
||||
private function parseValues(string $valuesRaw, int $columnCount): array
|
||||
{
|
||||
$rows = [];
|
||||
$currentRow = [];
|
||||
$buffer = '';
|
||||
$inString = false;
|
||||
$escape = false;
|
||||
$valueIsString = false;
|
||||
$inRow = false;
|
||||
$length = strlen($valuesRaw);
|
||||
|
||||
for ($i = 0; $i < $length; $i++) {
|
||||
$ch = $valuesRaw[$i];
|
||||
|
||||
if ($inString) {
|
||||
if ($escape) {
|
||||
$buffer .= $this->unescapeChar($ch);
|
||||
$escape = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === '\\\\') {
|
||||
$escape = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === '\'') {
|
||||
$inString = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
$buffer .= $ch;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === '(') {
|
||||
$inRow = true;
|
||||
$currentRow = [];
|
||||
$buffer = '';
|
||||
$valueIsString = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === '\'') {
|
||||
$inString = true;
|
||||
$valueIsString = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === ',' && $inRow) {
|
||||
$currentRow[] = $this->convertValue($buffer, $valueIsString);
|
||||
$buffer = '';
|
||||
$valueIsString = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($ch === ')' && $inRow) {
|
||||
$currentRow[] = $this->convertValue($buffer, $valueIsString);
|
||||
if (count($currentRow) === $columnCount) {
|
||||
$rows[] = $currentRow;
|
||||
}
|
||||
$buffer = '';
|
||||
$valueIsString = false;
|
||||
$inRow = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($inRow) {
|
||||
$buffer .= $ch;
|
||||
}
|
||||
}
|
||||
|
||||
return $rows;
|
||||
}
|
||||
|
||||
private function convertValue(string $buffer, bool $valueIsString): mixed
|
||||
{
|
||||
$value = trim($buffer);
|
||||
|
||||
if (!$valueIsString) {
|
||||
if ($value === '' || strtoupper($value) === 'NULL') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (is_numeric($value)) {
|
||||
return $value + 0;
|
||||
}
|
||||
}
|
||||
|
||||
return $value;
|
||||
}
|
||||
|
||||
private function unescapeChar(string $ch): string
|
||||
{
|
||||
return match ($ch) {
|
||||
'n' => "\n",
|
||||
'r' => "\r",
|
||||
't' => "\t",
|
||||
'0' => "\0",
|
||||
'Z' => "\x1A",
|
||||
default => $ch,
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user