Enhance Docker and PowerShell scripts for improved functionality and maintainability
All checks were successful
Publish FaceAI Container / publish (push) Successful in 6m52s
All checks were successful
Publish FaceAI Container / publish (push) Successful in 6m52s
- Updated Dockerfile to include default MySQL client for better database interaction. - Modified entrypoint.sh to support additional workspace for legacy applications and added MySQL readiness check before startup. - Enhanced PowerShell script for trimming MySQL dumps to include overlay dumps and improved error handling for missing race and user IDs. - Added new image files and face encoding pickles for various projects, ensuring comprehensive data availability. - Removed outdated face encoding pickle from PISA directory to maintain data relevance. Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
parent
c227fce036
commit
dd7d4c865b
54 changed files with 492 additions and 144 deletions
|
|
@ -1,6 +1,9 @@
|
|||
param(
|
||||
[string]$SourceDump = (Join-Path $PSScriptRoot '..\db\dump-pg-202604211927.pretrim-backup.sql'),
|
||||
[string]$OutputDump = (Join-Path $PSScriptRoot '..\db\dump-pg-202604211927.trimmed.sql'),
|
||||
[string]$SourceDump = (Join-Path $PSScriptRoot '..\db\pg-model-seed-trimmed-20260421.sql'),
|
||||
[string]$OverlayDump = (Join-Path $PSScriptRoot '..\db\pg-local-model-fixtures-overlay-20260422.sql'),
|
||||
[string]$OutputDump = (Join-Path $PSScriptRoot '..\db\pg-local-purpose-seed-20260422.sql'),
|
||||
[int[]]$KeepRaceIds = @(1018547, 1018557),
|
||||
[int[]]$KeepUserIds = @(2),
|
||||
[string]$ContainerName = 'regalami-dump-trim-mysql-temp',
|
||||
[string]$VolumeName = 'regalami-dump-trim-mysql-temp-data',
|
||||
[string]$DatabaseName = 'pgtrim',
|
||||
|
|
@ -44,12 +47,9 @@ function Wait-ForMysqlReady {
|
|||
)
|
||||
|
||||
for ($attempt = 0; $attempt -lt 180; $attempt++) {
|
||||
$portOutput = & docker exec $Name mysql -N -B -uroot -p$Password -e "SELECT @@port;" 2>$null
|
||||
if ($LASTEXITCODE -eq 0) {
|
||||
$reportedPort = ($portOutput | Select-Object -First 1).Trim()
|
||||
if ($reportedPort -eq '3306') {
|
||||
return
|
||||
}
|
||||
$logs = & docker logs $Name 2>&1
|
||||
if ($LASTEXITCODE -eq 0 -and ($logs -join "`n") -match 'ready for connections.*port: 3306') {
|
||||
return
|
||||
}
|
||||
|
||||
Start-Sleep -Seconds 2
|
||||
|
|
@ -65,7 +65,7 @@ function Invoke-MysqlQuery {
|
|||
[switch]$SkipDatabase
|
||||
)
|
||||
|
||||
$dockerArgs = @('exec', $ContainerName, 'mysql', '-N', '-B', '-uroot', "-p$RootPassword")
|
||||
$dockerArgs = @('exec', '-e', "MYSQL_PWD=$RootPassword", $ContainerName, 'mysql', '-N', '-B', '-uroot')
|
||||
if (-not $SkipDatabase) {
|
||||
$dockerArgs += @('-D', $DatabaseName)
|
||||
}
|
||||
|
|
@ -93,9 +93,18 @@ function Quote-Identifier {
|
|||
return '`' + $Name.Replace('`', '``') + '`'
|
||||
}
|
||||
|
||||
function Quote-SqlLiteral {
|
||||
param([string]$Value)
|
||||
return "'" + $Value.Replace("'", "''") + "'"
|
||||
function ConvertTo-SqlIntList {
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[int[]]$Values
|
||||
)
|
||||
|
||||
$dedupedValues = $Values | Sort-Object -Unique
|
||||
if (-not $dedupedValues -or $dedupedValues.Count -eq 0) {
|
||||
throw 'At least one numeric keep id is required.'
|
||||
}
|
||||
|
||||
return ($dedupedValues | ForEach-Object { [string]$_ }) -join ','
|
||||
}
|
||||
|
||||
if (-not (Test-Path $SourceDump)) {
|
||||
|
|
@ -105,32 +114,62 @@ if (-not (Test-Path $SourceDump)) {
|
|||
$dbDirectory = Split-Path -Parent (Resolve-Path $SourceDump).Path
|
||||
$sourceFileName = Split-Path -Leaf $SourceDump
|
||||
$outputFileName = Split-Path -Leaf $OutputDump
|
||||
$overlayResolvedPath = $null
|
||||
$overlayFileName = ''
|
||||
$overlayDirectory = $dbDirectory
|
||||
$overlayMount = @()
|
||||
if ($OverlayDump -and (Test-Path $OverlayDump)) {
|
||||
$overlayResolvedPath = (Resolve-Path $OverlayDump).Path
|
||||
$overlayFileName = Split-Path -Leaf $overlayResolvedPath
|
||||
$overlayDirectory = Split-Path -Parent $overlayResolvedPath
|
||||
if ($overlayDirectory -ne $dbDirectory) {
|
||||
$overlayMount = @('-v', "${overlayDirectory}:/workspace/overlay")
|
||||
}
|
||||
}
|
||||
$importErrorLogFileName = [System.IO.Path]::GetFileNameWithoutExtension($sourceFileName) + '.import-errors.log'
|
||||
$importErrorLogPath = Join-Path $dbDirectory $importErrorLogFileName
|
||||
$sourceSizeBytes = (Get-Item $SourceDump).Length
|
||||
$keepRaceListSql = ConvertTo-SqlIntList -Values $KeepRaceIds
|
||||
$keepUserListSql = ConvertTo-SqlIntList -Values $KeepUserIds
|
||||
|
||||
Write-Host "Using source dump: $SourceDump"
|
||||
Write-Host "Source size: $sourceSizeBytes bytes"
|
||||
Write-Host "Import error log: $importErrorLogPath"
|
||||
if ($overlayResolvedPath) {
|
||||
Write-Host "Using overlay dump: $overlayResolvedPath"
|
||||
}
|
||||
Write-Host "KeepRaceIdsRequested=$keepRaceListSql"
|
||||
Write-Host "KeepUserIdsRequested=$keepUserListSql"
|
||||
|
||||
try {
|
||||
& docker rm -f $ContainerName 1>$null 2>$null
|
||||
& docker volume rm $VolumeName 1>$null 2>$null
|
||||
|
||||
Invoke-DockerCapture -DockerArgs @(
|
||||
$dockerRunArgs = @(
|
||||
'run', '-d', '--name', $ContainerName,
|
||||
'-e', "MYSQL_ROOT_PASSWORD=$RootPassword",
|
||||
'-e', 'MYSQL_ROOT_HOST=%',
|
||||
'-v', "${dbDirectory}:/workspace/db",
|
||||
'-v', "${VolumeName}:/var/lib/mysql",
|
||||
'-v', "${VolumeName}:/var/lib/mysql"
|
||||
)
|
||||
if ($overlayMount.Count -gt 0) {
|
||||
$dockerRunArgs += $overlayMount
|
||||
}
|
||||
$dockerRunArgs += @(
|
||||
'mysql:8.4',
|
||||
'--max_allowed_packet=1G',
|
||||
'--net_read_timeout=600',
|
||||
'--net_write_timeout=600'
|
||||
) | Out-Null
|
||||
)
|
||||
|
||||
Invoke-DockerCapture -DockerArgs $dockerRunArgs | Out-Null
|
||||
Write-Host 'Temporary MySQL container started.'
|
||||
|
||||
Wait-ForMysqlReady -Name $ContainerName -Password $RootPassword
|
||||
Write-Host 'Temporary MySQL is ready.'
|
||||
|
||||
Invoke-MysqlQuery -SkipDatabase -Query "DROP DATABASE IF EXISTS $DatabaseName; CREATE DATABASE $DatabaseName CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;" | Out-Null
|
||||
Write-Host 'Temporary database created.'
|
||||
|
||||
Write-Host "Importing source dump into temporary MySQL database..."
|
||||
if (Test-Path $importErrorLogPath) {
|
||||
|
|
@ -141,6 +180,23 @@ try {
|
|||
"mysql --force -uroot -p$RootPassword $DatabaseName < /workspace/db/$sourceFileName 2> /workspace/db/$importErrorLogFileName"
|
||||
)
|
||||
|
||||
if ($overlayResolvedPath) {
|
||||
$overlayContainerPath = if ($overlayDirectory -eq $dbDirectory) {
|
||||
"/workspace/db/$overlayFileName"
|
||||
}
|
||||
else {
|
||||
"/workspace/overlay/$overlayFileName"
|
||||
}
|
||||
|
||||
Write-Host "Importing overlay dump into temporary MySQL database..."
|
||||
Invoke-DockerQuiet -DockerArgs @(
|
||||
'exec', $ContainerName, 'sh', '-lc',
|
||||
"mysql --force -uroot -p$RootPassword $DatabaseName < $overlayContainerPath"
|
||||
)
|
||||
}
|
||||
|
||||
$beforeGara = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM gara;')
|
||||
$beforePuntoFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM punto_foto;')
|
||||
$beforeFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM foto;')
|
||||
$beforeLogFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM log_foto;')
|
||||
$usersExists = (Get-SqlScalar -SkipDatabase -Query "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '$DatabaseName' AND table_name = 'users';") -eq '1'
|
||||
|
|
@ -149,45 +205,19 @@ try {
|
|||
$beforeUsers = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM users;')
|
||||
}
|
||||
|
||||
$garaDateColumn = Get-SqlScalar -SkipDatabase -Query @"
|
||||
SELECT COLUMN_NAME
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = '$DatabaseName'
|
||||
AND table_name = 'gara'
|
||||
AND COLUMN_NAME IN ('dataGaraInizio', 'dataGaraFine', 'data', 'createTmst', 'lastUpdTmst')
|
||||
ORDER BY FIELD(COLUMN_NAME, 'dataGaraInizio', 'dataGaraFine', 'data', 'createTmst', 'lastUpdTmst')
|
||||
LIMIT 1;
|
||||
"@
|
||||
if (-not $garaDateColumn) {
|
||||
throw 'Could not determine race date column from gara table.'
|
||||
}
|
||||
|
||||
$usersDateColumn = ''
|
||||
if ($usersExists) {
|
||||
$usersDateColumn = Get-SqlScalar -SkipDatabase -Query @"
|
||||
SELECT COLUMN_NAME
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = '$DatabaseName'
|
||||
AND table_name = 'users'
|
||||
AND COLUMN_NAME IN ('dataInserimento', 'dataInizioVld', 'lastUpdTmst')
|
||||
ORDER BY FIELD(COLUMN_NAME, 'dataInserimento', 'dataInizioVld', 'lastUpdTmst')
|
||||
LIMIT 1;
|
||||
"@
|
||||
if (-not $usersDateColumn) {
|
||||
$usersDateColumn = 'id_users'
|
||||
}
|
||||
}
|
||||
|
||||
$garaDateExpr = Quote-Identifier $garaDateColumn
|
||||
$usersDateExpr = Quote-Identifier $usersDateColumn
|
||||
|
||||
Invoke-MysqlQuery -Query @"
|
||||
DROP TABLE IF EXISTS keep_gara_ids;
|
||||
CREATE TABLE keep_gara_ids AS
|
||||
SELECT id_gara
|
||||
FROM gara
|
||||
ORDER BY $garaDateExpr DESC, id_gara DESC
|
||||
LIMIT 10;
|
||||
WHERE id_gara IN ($keepRaceListSql)
|
||||
ORDER BY FIELD(id_gara, $keepRaceListSql);
|
||||
|
||||
DROP TABLE IF EXISTS keep_punto_foto_ids;
|
||||
CREATE TABLE keep_punto_foto_ids AS
|
||||
SELECT id_puntoFoto
|
||||
FROM punto_foto
|
||||
WHERE id_gara IN (SELECT id_gara FROM keep_gara_ids);
|
||||
|
||||
DROP TABLE IF EXISTS keep_foto_ids;
|
||||
CREATE TABLE keep_foto_ids AS
|
||||
|
|
@ -196,17 +226,50 @@ FROM foto
|
|||
WHERE id_gara IN (SELECT id_gara FROM keep_gara_ids);
|
||||
"@ | Out-Null
|
||||
|
||||
$keptGaraIds = Invoke-MysqlQuery -Query 'SELECT id_gara FROM keep_gara_ids ORDER BY id_gara;'
|
||||
$missingRaceIds = @()
|
||||
foreach ($requestedRaceId in ($KeepRaceIds | Sort-Object -Unique)) {
|
||||
if (-not (($keptGaraIds | ForEach-Object { $_.Trim() }) -contains [string]$requestedRaceId)) {
|
||||
$missingRaceIds += $requestedRaceId
|
||||
}
|
||||
}
|
||||
if ($missingRaceIds.Count -gt 0) {
|
||||
throw "One or more requested race ids are missing from the imported data: $($missingRaceIds -join ',')"
|
||||
}
|
||||
|
||||
$cleanupStats = [System.Collections.Generic.List[string]]::new()
|
||||
|
||||
Invoke-MysqlQuery -Query 'DELETE FROM log_foto;' | Out-Null
|
||||
$cleanupStats.Add('log_foto:deleted-all')
|
||||
|
||||
$garaDependentTables = Invoke-MysqlQuery -SkipDatabase -Query @"
|
||||
SELECT table_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = '$DatabaseName'
|
||||
AND column_name = 'id_gara'
|
||||
AND table_name NOT IN ('gara', 'foto', 'punto_foto', 'keep_gara_ids', 'keep_punto_foto_ids', 'keep_foto_ids', 'keep_user_ids')
|
||||
ORDER BY table_name;
|
||||
"@
|
||||
|
||||
foreach ($tableName in $garaDependentTables) {
|
||||
if (-not $tableName) {
|
||||
continue
|
||||
}
|
||||
$trimmedTableName = $tableName.Trim()
|
||||
$quotedTable = Quote-Identifier $trimmedTableName
|
||||
$deletedRows = [int64](Get-SqlScalar -Query "SELECT COUNT(*) FROM $quotedTable WHERE id_gara IS NOT NULL AND id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);")
|
||||
if ($deletedRows -gt 0) {
|
||||
Invoke-MysqlQuery -Query "DELETE FROM $quotedTable WHERE id_gara IS NOT NULL AND id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);" | Out-Null
|
||||
$cleanupStats.Add("$($trimmedTableName):$deletedRows")
|
||||
}
|
||||
}
|
||||
|
||||
$fotoDependentTables = Invoke-MysqlQuery -SkipDatabase -Query @"
|
||||
SELECT table_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = '$DatabaseName'
|
||||
AND column_name = 'id_foto'
|
||||
AND table_name NOT IN ('foto', 'log_foto', 'keep_foto_ids', 'keep_gara_ids')
|
||||
AND table_name NOT IN ('foto', 'log_foto', 'keep_foto_ids', 'keep_gara_ids', 'keep_punto_foto_ids', 'keep_user_ids')
|
||||
ORDER BY table_name;
|
||||
"@
|
||||
|
||||
|
|
@ -229,22 +292,46 @@ ORDER BY table_name;
|
|||
$cleanupStats.Add("foto:$deletedFotoRows")
|
||||
}
|
||||
|
||||
$deletedPuntoFotoRows = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM punto_foto WHERE id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);')
|
||||
if ($deletedPuntoFotoRows -gt 0) {
|
||||
Invoke-MysqlQuery -Query 'DELETE FROM punto_foto WHERE id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);' | Out-Null
|
||||
$cleanupStats.Add("punto_foto:$deletedPuntoFotoRows")
|
||||
}
|
||||
|
||||
$deletedGaraRows = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM gara WHERE id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);')
|
||||
if ($deletedGaraRows -gt 0) {
|
||||
Invoke-MysqlQuery -Query 'DELETE FROM gara WHERE id_gara NOT IN (SELECT id_gara FROM keep_gara_ids);' | Out-Null
|
||||
$cleanupStats.Add("gara:$deletedGaraRows")
|
||||
}
|
||||
|
||||
$keptUserIds = @()
|
||||
if ($usersExists) {
|
||||
Invoke-MysqlQuery -Query @"
|
||||
DROP TABLE IF EXISTS keep_user_ids;
|
||||
CREATE TABLE keep_user_ids AS
|
||||
SELECT id_users
|
||||
FROM users
|
||||
ORDER BY $usersDateExpr ASC, id_users ASC
|
||||
LIMIT 5;
|
||||
WHERE id_users IN ($keepUserListSql)
|
||||
ORDER BY FIELD(id_users, $keepUserListSql);
|
||||
"@ | Out-Null
|
||||
|
||||
$keptUserIds = Invoke-MysqlQuery -Query 'SELECT id_users FROM keep_user_ids ORDER BY id_users;'
|
||||
$missingUserIds = @()
|
||||
foreach ($requestedUserId in ($KeepUserIds | Sort-Object -Unique)) {
|
||||
if (-not (($keptUserIds | ForEach-Object { $_.Trim() }) -contains [string]$requestedUserId)) {
|
||||
$missingUserIds += $requestedUserId
|
||||
}
|
||||
}
|
||||
if ($missingUserIds.Count -gt 0) {
|
||||
throw "One or more requested user ids are missing from the imported data: $($missingUserIds -join ',')"
|
||||
}
|
||||
|
||||
$usersDependentTables = Invoke-MysqlQuery -SkipDatabase -Query @"
|
||||
SELECT table_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = '$DatabaseName'
|
||||
AND column_name = 'id_users'
|
||||
AND table_name NOT IN ('users', 'keep_user_ids')
|
||||
AND table_name NOT IN ('users', 'keep_user_ids', 'keep_gara_ids', 'keep_punto_foto_ids', 'keep_foto_ids')
|
||||
ORDER BY table_name;
|
||||
"@
|
||||
|
||||
|
|
@ -268,6 +355,8 @@ ORDER BY table_name;
|
|||
}
|
||||
}
|
||||
|
||||
$afterGara = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM gara;')
|
||||
$afterPuntoFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM punto_foto;')
|
||||
$afterFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM foto;')
|
||||
$afterLogFoto = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM log_foto;')
|
||||
$afterUsers = 0
|
||||
|
|
@ -275,26 +364,28 @@ ORDER BY table_name;
|
|||
$afterUsers = [int64](Get-SqlScalar -Query 'SELECT COUNT(*) FROM users;')
|
||||
}
|
||||
|
||||
$keptGaraIds = Invoke-MysqlQuery -Query "SELECT id_gara FROM keep_gara_ids ORDER BY $garaDateExpr DESC, id_gara DESC;"
|
||||
|
||||
if (Test-Path $OutputDump) {
|
||||
Remove-Item $OutputDump -Force
|
||||
}
|
||||
|
||||
Write-Host "Exporting trimmed dump..."
|
||||
Write-Host "Exporting curated dump..."
|
||||
Invoke-DockerQuiet -DockerArgs @(
|
||||
'exec', $ContainerName, 'sh', '-lc',
|
||||
"mysqldump -uroot -p$RootPassword --default-character-set=utf8mb4 --single-transaction --routines --triggers --set-gtid-purged=OFF $DatabaseName > /workspace/db/$outputFileName"
|
||||
)
|
||||
|
||||
if (-not (Test-Path $OutputDump)) {
|
||||
throw "Trimmed dump was not created: $OutputDump"
|
||||
throw "Curated dump was not created: $OutputDump"
|
||||
}
|
||||
|
||||
$outputSizeBytes = (Get-Item $OutputDump).Length
|
||||
|
||||
Write-Host "OriginalSizeBytes=$sourceSizeBytes"
|
||||
Write-Host "TrimmedSizeBytes=$outputSizeBytes"
|
||||
Write-Host "CuratedSizeBytes=$outputSizeBytes"
|
||||
Write-Host "BeforeGara=$beforeGara"
|
||||
Write-Host "AfterGara=$afterGara"
|
||||
Write-Host "BeforePuntoFoto=$beforePuntoFoto"
|
||||
Write-Host "AfterPuntoFoto=$afterPuntoFoto"
|
||||
Write-Host "BeforeFoto=$beforeFoto"
|
||||
Write-Host "AfterFoto=$afterFoto"
|
||||
Write-Host "BeforeLogFoto=$beforeLogFoto"
|
||||
|
|
@ -302,6 +393,9 @@ ORDER BY table_name;
|
|||
Write-Host "BeforeUsers=$beforeUsers"
|
||||
Write-Host "AfterUsers=$afterUsers"
|
||||
Write-Host ("KeptGaraIds=" + (($keptGaraIds | ForEach-Object { $_.Trim() }) -join ','))
|
||||
if ($usersExists) {
|
||||
Write-Host ("KeptUserIds=" + (($keptUserIds | ForEach-Object { $_.Trim() }) -join ','))
|
||||
}
|
||||
Write-Host ("DependencyCleanup=" + (($cleanupStats | Sort-Object) -join ';'))
|
||||
if (Test-Path $importErrorLogPath) {
|
||||
$importErrorCount = (Get-Item $importErrorLogPath).Length
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue