From: U-QUAN-W4\Administrator Date: Sun, 21 Mar 2010 09:18:35 +0000 (+0800) Subject: new options & bug fix X-Git-Url: http://git.postgresql.org/gitweb/?a=commitdiff_plain;h=11e09eb62c643c7102ca24fb3625b42c8aedf63a;p=users%2Fquan74%2Fpgadmin-import.git new options & bug fix --- diff --git a/pgadmin/frm/frmImport.cpp b/pgadmin/frm/frmImport.cpp index 4bd37023b..d878eaa41 100755 --- a/pgadmin/frm/frmImport.cpp +++ b/pgadmin/frm/frmImport.cpp @@ -74,6 +74,7 @@ //wzdFileOptsPage #define chkFileSkipEmptyRow CTRL_CHECKBOX("chkFileSkipEmptyRow") #define txtFileOptImpRegEx CTRL_TEXT("txtFileOptImpRegEx") +#define txtFileOptStartWith CTRL_TEXT("txtFileOptStartWith") #define spinFileSkipFirstRows CTRL_SPIN("spinFileSkipFirstRows") #define rdbFileImpAllRows CTRL_RADIOBUTTON("rdbFileImpAllRows") #define rdbFileImpOnlyRows CTRL_RADIOBUTTON("rdbFileImpOnlyRows") @@ -96,6 +97,7 @@ #define chcImpDest CTRL_CHOICE("chcImpDest") #define txtDestFilePath CTRL_TEXT("txtDestFilePath") #define btnDestFileBrowse CTRL_BUTTON("btnDestFileBrowse") +#define chkDestFileAppend CTRL_CHECKBOX("chkDestFileAppend") #define spinCommitEvery CTRL_SPIN("spinCommitEvery") #define spinStopOnError CTRL_SPIN("spinStopOnError") #define rdbLogToScreen CTRL_RADIOBUTTON("rdbLogToScreen") @@ -103,6 +105,7 @@ #define rdbLogToScrFile CTRL_RADIOBUTTON("rdbLogToScrFile") #define txtLogFilePath CTRL_TEXT("txtLogFilePath") #define btnLogFileBrowse CTRL_BUTTON("btnLogFileBrowse") +#define chkLogFileAppend CTRL_CHECKBOX("chkLogFileAppend") #define rdbLogVerbose CTRL_RADIOBUTTON("rdbLogVerbose") #define rdbLogSummary CTRL_RADIOBUTTON("rdbLogSummary") #define rdbLogSilent CTRL_RADIOBUTTON("rdbLogSilent") @@ -125,15 +128,17 @@ BEGIN_EVENT_TABLE(frmImport, wxWizard) // wizard event EVT_WIZARD_PAGE_CHANGING(XRCID("frmImport"), frmImport::OnPageChanging) EVT_WIZARD_CANCEL(XRCID("frmImport"), frmImport::OnWizardCancel) + // TODO: check whether finish button has bug that can't be disabled? + // if or not add a new method to catch finish wizard event. // Source Type page EVT_RADIOBUTTON(XRCID("rdbSrcTextFile"), frmImport::OnSourceType) EVT_RADIOBUTTON(XRCID("rdbSrcArchFile"), frmImport::OnSourceType) EVT_RADIOBUTTON(XRCID("rdbSrcODBC"), frmImport::OnSourceType) EVT_RADIOBUTTON(XRCID("rdbSrcPgSQL"), frmImport::OnSourceType) // Text file page - EVT_BUTTON(XRCID("btnTextFileBrowse"), frmImport::OnTextFileBrowse) - EVT_CHOICE(XRCID("chcTextFileEncoding"), frmImport::OnTextFileEncoding) - EVT_BUTTON(XRCID("btnTextFilePreviewMore"), frmImport::OnTextFilePreviewMore) + EVT_BUTTON(XRCID("btnTextFileBrowse"), frmImport::OnTextFileBrowse) + EVT_CHOICE(XRCID("chcTextFileEncoding"), frmImport::OnTextFileEncoding) + EVT_BUTTON(XRCID("btnTextFilePreviewMore"), frmImport::OnTextFilePreviewMore) // Archive file page EVT_BUTTON(XRCID("btnArchFileBrowse"), frmImport::OnArchFileBrowse) EVT_CHOICE(XRCID("chcArchFileEncoding"), frmImport::OnArchFileEncoding) @@ -333,6 +338,8 @@ frmImport::~frmImport() if (m_fileoptimpregex) delete m_fileoptimpregex; + if (m_fileoptstartwith) + delete m_fileoptstartwith; if (m_booltruevals) delete m_booltruevals; @@ -449,6 +456,7 @@ void frmImport::Init() m_srcfile = NULL; m_fileoptimpregex = NULL; + m_fileoptstartwith = NULL; m_booltruevals = NULL; m_boolfalsevals = NULL; @@ -554,6 +562,11 @@ void frmImport::OnPageChanging(wxWizardEvent& ev) delete m_fileoptimpregex; m_fileoptimpregex = NULL; } + if (m_fileoptstartwith) + { + delete m_fileoptstartwith; + m_fileoptstartwith = NULL; + } } if ((m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE)) @@ -1289,11 +1302,13 @@ void frmImport::OnDestImpDest(wxCommandEvent &ev) { txtDestFilePath->Enable(); btnDestFileBrowse->Enable(); + chkDestFileAppend->Enable(); } else { txtDestFilePath->Disable(); btnDestFileBrowse->Disable(); + chkDestFileAppend->Disable(); } } @@ -1316,7 +1331,7 @@ void frmImport::OnDestFileBrowse(wxCommandEvent &ev) bool frmImport::OnDestFilePathChange(const wxString& filepath) { - if (wxFileExists(filepath)) + if (!chkDestFileAppend->GetValue() && wxFileExists(filepath)) { wxString msg; msg.Printf(_("The output file: \n\n%s\n\nalready exists. Do you want to overwrite it?"), filepath.c_str()); @@ -1334,11 +1349,13 @@ void frmImport::OnDestLogTo(wxCommandEvent &ev) { txtLogFilePath->Disable(); btnLogFileBrowse->Disable(); + chkLogFileAppend->Disable(); } else { txtLogFilePath->Enable(); btnLogFileBrowse->Enable(); + chkLogFileAppend->Enable(); } } @@ -1361,7 +1378,7 @@ void frmImport::OnLogFileBrowse(wxCommandEvent &ev) bool frmImport::OnLogFilePathChange(const wxString& filepath) { - if (wxFileExists(filepath)) + if (!chkLogFileAppend->GetValue() && wxFileExists(filepath)) { wxString msg; msg.Printf(_("The log file: \n\n%s\n\nalready exists. Do you want to overwrite it?"), filepath.c_str()); @@ -1465,11 +1482,12 @@ void frmImport::ImportData() int impmode = (int)chcImportMode->GetClientData(chcImportMode->GetSelection()); int impdest = (int)chcImpDest->GetClientData(chcImpDest->GetSelection()); + bool fileopened; if (m_procstatus==IMPPREPARED) { txtLogMessages->Clear(); - if ((!m_destfile && impdest==IMPDESTFCOPY) || impdest==IMPDESTFINS || impdest==IMPDESTFTEST || impdest==IMPDESTFILE) + if (!m_destfile && (impdest==IMPDESTFCOPY || impdest==IMPDESTFINS || impdest==IMPDESTFTEST || impdest==IMPDESTFILE)) { wxString destfilename = txtDestFilePath->GetValue(); if (wxFile::Exists(destfilename) && !wxFile::Access(destfilename, wxFile::write)) @@ -1481,7 +1499,11 @@ void frmImport::ImportData() return; } m_destfile = new wxFile; - if (!m_destfile->Open(destfilename, wxFile::write)) + if (chkDestFileAppend->GetValue()) + fileopened = m_destfile->Open(destfilename, wxFile::write_append); + else + fileopened = m_destfile->Open(destfilename, wxFile::write); + if (!fileopened) { delete m_destfile; m_destfile = NULL; @@ -1505,7 +1527,11 @@ void frmImport::ImportData() return; } m_logfile = new wxFile; - if (!m_logfile->Open(logfilename, wxFile::write)) + if (chkLogFileAppend->GetValue()) + fileopened = m_logfile->Open(logfilename, wxFile::write_append); + else + fileopened = m_logfile->Open(logfilename, wxFile::write); + if (!fileopened) { delete m_logfile; m_logfile = NULL; @@ -1718,6 +1744,7 @@ void frmImport::ImportData() m_imperrnum = 0; m_improwsnum = 0; + m_skiprowsnum = 0; m_transrowsnum = 0; RandomSavepointName(m_imptransname); @@ -1727,7 +1754,7 @@ void frmImport::ImportData() { m_imppausedts += wxDateTime::UNow() - m_imppausedt; - //m_procstatus = IMPRUNNING; + m_procstatus = IMPRUNNING; btnImpStart->SetLabel(_("&Pause")); wzdctrlbtn = (wxButton*)FindWindowById(wxID_BACKWARD); wzdctrlbtn->Disable(); @@ -1807,8 +1834,8 @@ void frmImport::ImportData() skipfirstrows = false; } - int commitevery = spinCommitEvery->GetValue(); - int stoponerror = spinStopOnError->GetValue(); + unsigned int commitevery = spinCommitEvery->GetValue(); + unsigned int stoponerror = spinStopOnError->GetValue(); #if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) SQLSMALLINT stmtcolsnum; @@ -1837,18 +1864,17 @@ void frmImport::ImportData() wxString **linevals; int valsnum; wxDateTime clienttime; - wxTimeSpan elapsedspan; - int avgrowtime; wxString impdatalist, insstmt, rowlog; wxChar imptranssn[32]; - bool copybolcking = false; - bool rowdataisok; + bool needflush = false; + bool rowdataisok, lineisskipped = false; int commres; while (true) { if (m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE) { + lineisskipped = false; linedata.Empty(); linevals = NULL; if (!m_srcfile->NextLine(linedata, linevals, valsnum)) @@ -1907,7 +1933,7 @@ void frmImport::ImportData() } } - continue; + lineisskipped = true; } if (skipemptyline && linedata.IsEmpty()) @@ -1926,7 +1952,7 @@ void frmImport::ImportData() } } - continue; + lineisskipped = true; } if (m_fileoptimpregex && !m_fileoptimpregex->Matches(linedata)) @@ -1945,7 +1971,26 @@ void frmImport::ImportData() } } - continue; + lineisskipped = true; + } + + if (m_fileoptstartwith && !linedata.StartsWith(*m_fileoptstartwith)) + { + if (loginfo) + { + if (logtoscr) + { + txtLogMessages->AppendText(_("Skip not match start with row.")); + txtLogMessages->AppendText(END_OF_LINE); + } + if (m_logfile) + { + m_logfile->Write(_("Skip not match start with row.")); + m_logfile->Write(END_OF_LINE); + } + } + + lineisskipped = true; } } #if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) @@ -2081,11 +2126,15 @@ void frmImport::ImportData() else break; - impdatalist.Empty(); - clienttime = wxDateTime::UNow(); - rowlog.Empty(); + if (!lineisskipped) + { + impdatalist.Empty(); + clienttime = wxDateTime::UNow(); + rowlog.Empty(); + } - if (m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE) + if (lineisskipped); + else if (m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE) rowdataisok = GenCSVSQLData(impdatalist, rowlog, linevals, valsnum, clienttime, impmode, fmtstyle); #if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) else if (m_datasrctype==SRCODBCPAGE) @@ -2094,45 +2143,170 @@ void frmImport::ImportData() else rowdataisok = false; - if (rowdataisok) + if (needflush) { if (impmode==IMPORTMODECOPY) { - if (copybolcking) + if (m_impconn->FlushData()) { - if (m_impconn->FlushData()) + if (loginfo) { - if (loginfo) + if (logtoscr) + { + txtLogMessages->AppendText(_("Data flushed.")); + txtLogMessages->AppendText(END_OF_LINE); + } + if (m_logfile) + { + m_logfile->Write(_("Data flushed.")); + m_logfile->Write(END_OF_LINE); + } + } + } + else + { + if (logerror) + { + if (logtoscr) + txtLogMessages->AppendText(_("Failed to flush data: ") + m_impconn->GetLastError()); + if (m_logfile) + m_logfile->Write(_("Failed to flush data: ") + m_impconn->GetLastError()); + } + m_procstatus = IMPERRORSTOP; + m_imperrnum++; + break; + } + } + else + { + insstmt.Empty(); + swprintf(imptranssn, 32, wxT("%d"), m_improwsnum-1); + + if (m_impconn->GetAsyncResult()) + { + if (loginfo) + { + if (logtoscr) + { + txtLogMessages->AppendText(_("Succeeded.")); + txtLogMessages->AppendText(END_OF_LINE); + } + if (m_logfile) + { + m_logfile->Write(_("Succeeded.")); + m_logfile->Write(END_OF_LINE); + } + } + m_transrowsnum++; + } + else + { + if (logerror) + { + if (logtoscr) + txtLogMessages->AppendText(m_impconn->GetLastError()); + if (m_logfile) + m_logfile->Write(m_impconn->GetLastError()); + } + + insstmt.Append(wxT("ROLLBACK TO ")).Append(m_imptransname).Append(imptranssn).Append(wxT(';')); + m_imperrnum++; + } + + insstmt.Append(wxT("RELEASE SAVEPOINT ")).Append(m_imptransname).Append(imptranssn).Append(wxT(';')); + if (commitevery && m_transrowsnum>=commitevery) + { + if (impdest==IMPDESTTEST || impdest==IMPDESTFTEST) + { + insstmt.Append(wxT("ROLLBACK;")); + txtLogMessages->AppendText(_("Rollback and begin new transaction.")); + txtLogMessages->AppendText(END_OF_LINE); + if (loginfo && m_logfile) + { + m_logfile->Write(_("Rollback and begin new transaction.")); + m_logfile->Write(END_OF_LINE); + } + } + else + { + insstmt.Append(wxT("COMMIT;")); + txtLogMessages->AppendText(_("Commit and begin new transaction.")); + txtLogMessages->AppendText(END_OF_LINE); + if (loginfo && m_logfile) + { + m_logfile->Write(_("Commit and begin new transaction.")); + m_logfile->Write(END_OF_LINE); + } + } + insstmt.Append(wxT("BEGIN TRANSACTION;")); + m_transrowsnum = 0; + } + if (logstmt) + { + if (logtoscr) + { + txtLogMessages->AppendText(insstmt); + txtLogMessages->AppendText(END_OF_LINE); + } + if (m_logfile) + { + m_logfile->Write(insstmt); + m_logfile->Write(END_OF_LINE); + } + } + if (m_impconn->ExecuteVoid(insstmt, false)) + { + if (loginfo) + { + if (logtoscr) + { + txtLogMessages->AppendText(_("Succeeded.")); + txtLogMessages->AppendText(END_OF_LINE); + } + if (m_logfile) + { + m_logfile->Write(_("Succeeded.")); + m_logfile->Write(END_OF_LINE); + } + } + } + else + { + if (logerror) + { + if (!logstmt) { if (logtoscr) { - txtLogMessages->AppendText(_("Data flushed.")); + txtLogMessages->AppendText(insstmt); txtLogMessages->AppendText(END_OF_LINE); } if (m_logfile) { - m_logfile->Write(_("Data flushed.")); + m_logfile->Write(insstmt); m_logfile->Write(END_OF_LINE); } } - } - else - { - if (logerror) - { - if (logtoscr) - txtLogMessages->AppendText(_("Failed to flush data: ") + m_impconn->GetLastError()); - if (m_logfile) - m_logfile->Write(_("Failed to flush data: ") + m_impconn->GetLastError()); - } - m_procstatus = IMPERRORSTOP; - m_imperrnum++; - break; + if (logtoscr) + txtLogMessages->AppendText(m_impconn->GetLastError()); + if (m_logfile) + m_logfile->Write(m_impconn->GetLastError()); } - copybolcking = false; + m_procstatus = IMPERRORSTOP; + m_imperrnum++; + break; } + } + needflush = false; + } + + if (lineisskipped); + else if (rowdataisok) + { + if (impmode==IMPORTMODECOPY) + { if (loginfo && impdest!=IMPDESTFILE) { if (logtoscr) @@ -2211,137 +2385,12 @@ void frmImport::ImportData() m_logfile->Write(END_OF_LINE); } } - copybolcking = true; + needflush = true; } } else { - if (m_procstatus==IMPPAUSED) - m_procstatus = IMPRUNNING; - else if (impdest!=IMPDESTFILE && m_improwsnum) - { - insstmt.Empty(); - //_itow(m_improwsnum-1, imptranssn, 10); - swprintf(imptranssn, 32, wxT("%d"), m_improwsnum-1); - - if (m_impconn->GetAsyncResult()) - { - if (loginfo) - { - if (logtoscr) - { - txtLogMessages->AppendText(_("Succeeded.")); - txtLogMessages->AppendText(END_OF_LINE); - } - if (m_logfile) - { - m_logfile->Write(_("Succeeded.")); - m_logfile->Write(END_OF_LINE); - } - } - m_transrowsnum++; - } - else - { - if (logerror) - { - if (logtoscr) - txtLogMessages->AppendText(m_impconn->GetLastError()); - if (m_logfile) - m_logfile->Write(m_impconn->GetLastError()); - } - - insstmt.Append(wxT("ROLLBACK TO ")).Append(m_imptransname).Append(imptranssn).Append(wxT(';')); - m_imperrnum++; - } - - insstmt.Append(wxT("RELEASE SAVEPOINT ")).Append(m_imptransname).Append(imptranssn).Append(wxT(';')); - if (commitevery && m_transrowsnum>=commitevery) - { - if (impdest==IMPDESTTEST || impdest==IMPDESTFTEST) - { - insstmt.Append(wxT("ROLLBACK;")); - txtLogMessages->AppendText(_("Rollback and begin new transaction.")); - txtLogMessages->AppendText(END_OF_LINE); - if (loginfo && m_logfile) - { - m_logfile->Write(_("Rollback and begin new transaction.")); - m_logfile->Write(END_OF_LINE); - } - } - else - { - insstmt.Append(wxT("COMMIT;")); - txtLogMessages->AppendText(_("Commit and begin new transaction.")); - txtLogMessages->AppendText(END_OF_LINE); - if (loginfo && m_logfile) - { - m_logfile->Write(_("Commit and begin new transaction.")); - m_logfile->Write(END_OF_LINE); - } - } - insstmt.Append(wxT("BEGIN TRANSACTION;")); - m_transrowsnum = 0; - } - if (logstmt) - { - if (logtoscr) - { - txtLogMessages->AppendText(insstmt); - txtLogMessages->AppendText(END_OF_LINE); - } - if (m_logfile) - { - m_logfile->Write(insstmt); - m_logfile->Write(END_OF_LINE); - } - } - if (m_impconn->ExecuteVoid(insstmt, false)) - { - if (loginfo) - { - if (logtoscr) - { - txtLogMessages->AppendText(_("Succeeded.")); - txtLogMessages->AppendText(END_OF_LINE); - } - if (m_logfile) - { - m_logfile->Write(_("Succeeded.")); - m_logfile->Write(END_OF_LINE); - } - } - } - else - { - if (logerror) - { - if (!logstmt) - { - if (logtoscr) - { - txtLogMessages->AppendText(insstmt); - txtLogMessages->AppendText(END_OF_LINE); - } - if (m_logfile) - { - m_logfile->Write(insstmt); - m_logfile->Write(END_OF_LINE); - } - } - if (logtoscr) - txtLogMessages->AppendText(m_impconn->GetLastError()); - if (m_logfile) - m_logfile->Write(m_impconn->GetLastError()); - } - - m_procstatus = IMPERRORSTOP; - m_imperrnum++; - break; - } - } - if (impdest==IMPDESTFILE) { insstmt.Empty(); @@ -2350,7 +2399,6 @@ void frmImport::ImportData() } else { - //_itow(m_improwsnum, imptranssn, 10); swprintf(imptranssn, 32, wxT("%d"), m_improwsnum); insstmt.Empty(); @@ -2421,6 +2469,7 @@ void frmImport::ImportData() m_logfile->Write(END_OF_LINE); } } + needflush = true; } else { @@ -2498,45 +2547,19 @@ void frmImport::ImportData() m_imperrnum++; } - m_improwsnum++; + if (lineisskipped) + m_skiprowsnum++; + else + m_improwsnum++; if (logproc) - { - stxtInsertedRows->SetLabel(wxString::Format(_("%d row(s) inserted, %d error(s)."), m_improwsnum-m_imperrnum, m_imperrnum)); - elapsedspan = wxDateTime::UNow().Subtract(m_impstartdt).Subtract(m_imppausedts); - avgrowtime = elapsedspan.GetMilliseconds().ToLong()/m_improwsnum; - stxtElapsedTime->SetLabel(elapsedspan.Format(wxT("%H:%M:%S.%l"))); - stxtAvgRowTime->SetLabel(wxString::Format(_("%d ms"), avgrowtime)); - - if (m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE) - { - if (m_srcsize==wxInvalidOffset) - gaugeImp->SetValue(m_improwsnum%100); - else - { - gaugeImp->SetValue(m_srcfile->Tell()*1000/m_srcsize); - stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(elapsedspan.GetMilliseconds().ToLong()*((m_srcsize-m_srcfile->Tell())/m_srcfile->Tell())).Format(wxT("%H:%M:%S.%l"))); - } - } -#if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) - else if (m_datasrctype==SRCODBCPAGE) - { - if (m_srcsize==-1) - gaugeImp->SetValue(m_improwsnum%100); - else - { - gaugeImp->SetValue(m_improwsnum*1000/m_srcsize); - stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(avgrowtime*(m_srcsize-m_improwsnum)).Format(wxT("%H:%M:%S.%l"))); - } - } -#endif - } + RefreshProc(); if (logtoscr) if (txtLogMessages->GetLastPosition()>MAXLOGCOUNT) { txtLogMessages->Freeze(); - txtLogMessages->Remove(0, 1024); + txtLogMessages->Remove(0, 2048); txtLogMessages->Thaw(); } @@ -2557,7 +2580,7 @@ void frmImport::ImportData() if (impdest!=IMPDESTFILE && (m_procstatus==IMPPAUSING || m_procstatus==IMPSTOPING)) { - if (impmode==IMPORTMODECOPY && copybolcking) + if (impmode==IMPORTMODECOPY && needflush) { if (m_impconn->FlushData()) { @@ -2586,10 +2609,9 @@ void frmImport::ImportData() } } } - else if (impmode==IMPORTMODEINSERT) + else if (impmode==IMPORTMODEINSERT && needflush) { insstmt.Empty(); - //_itow(m_improwsnum-1, imptranssn, 10); swprintf(imptranssn, 32, wxT("%d"), m_improwsnum-1); if (m_impconn->GetAsyncResult()) @@ -2607,6 +2629,7 @@ void frmImport::ImportData() m_logfile->Write(END_OF_LINE); } } + m_transrowsnum++; } else { @@ -2685,8 +2708,6 @@ void frmImport::ImportData() { m_imppausedt = wxDateTime::UNow(); wxLog::GetActiveTarget()->SetLogLevel(settings->GetLogLevel()); - btnImpStart->Enable(); - btnImpStop->Enable(); btnImpStart->SetLabel(_("&Resume")); #if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) @@ -2695,24 +2716,13 @@ void frmImport::ImportData() #endif if (!logproc) - { - if (m_srcsize==-1) - gaugeImp->SetValue(m_improwsnum%100); - else - gaugeImp->SetValue(m_improwsnum*1000/m_srcsize); - - stxtInsertedRows->SetLabel(wxString::Format(_("%d row(s) inserted, %d error(s)."), m_improwsnum-m_imperrnum, m_imperrnum)); - elapsedspan = wxDateTime::UNow().Subtract(m_impstartdt).Subtract(m_imppausedts); - avgrowtime = elapsedspan.GetMilliseconds().ToLong()/m_improwsnum; - stxtElapsedTime->SetLabel(elapsedspan.Format(wxT("%H:%M:%S.%l"))); - if (m_srcsize!=-1) - stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(avgrowtime*(m_srcsize-m_improwsnum)).Format(wxT("%H:%M:%S.%l"))); - stxtAvgRowTime->SetLabel(wxString::Format(_("%d ms"), avgrowtime)); - } + RefreshProc(); wzdctrlbtn = (wxButton*)FindWindowById(wxID_BACKWARD); wzdctrlbtn->Enable(); m_procstatus = IMPPAUSED; + btnImpStart->Enable(); + btnImpStop->Enable(); return; } @@ -2757,7 +2767,7 @@ void frmImport::ImportData() { if (impdest!=IMPDESTFILE) { - if (copybolcking) + if (needflush) { if (m_impconn->FlushData()) { @@ -2810,10 +2820,9 @@ void frmImport::ImportData() } else { - if (impdest!=IMPDESTFILE) + if (needflush && impdest!=IMPDESTFILE) { insstmt.Empty(); - //_itow(m_improwsnum-1, imptranssn, 10); swprintf(imptranssn, 32, wxT("%d"), m_improwsnum-1); if (m_impconn->GetAsyncResult()) @@ -2997,15 +3006,7 @@ void frmImport::ImportData() gaugeImp->SetValue(gaugeImp->GetRange()); if (!logproc && m_procstatus!=IMPERRORSTOP && m_procstatus!=IMPSTOPED) - { - stxtInsertedRows->SetLabel(wxString::Format(_("%d row(s) inserted, %d error(s)."), m_improwsnum-m_imperrnum, m_imperrnum)); - elapsedspan = wxDateTime::UNow().Subtract(m_impstartdt).Subtract(m_imppausedts); - avgrowtime = elapsedspan.GetMilliseconds().ToLong()/m_improwsnum; - stxtElapsedTime->SetLabel(elapsedspan.Format(wxT("%H:%M:%S.%l"))); - if (m_srcsize!=-1) - stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(avgrowtime*(m_srcsize-m_improwsnum)).Format(wxT("%H:%M:%S.%l"))); - stxtAvgRowTime->SetLabel(wxString::Format(_("%d ms"), avgrowtime)); - } + RefreshProc(false); btnImpStart->Disable(); btnImpStop->Disable(); @@ -3039,6 +3040,42 @@ void frmImport::ImportData() } +void frmImport::RefreshProc(bool incgauge) +{ + wxTimeSpan elapsedspan; + int avgrowtime; + + stxtInsertedRows->SetLabel(wxString::Format(_("%d row(s) inserted, %d error(s), %d skipped."), + m_improwsnum-m_imperrnum, m_imperrnum, m_skiprowsnum)); + elapsedspan = wxDateTime::UNow().Subtract(m_impstartdt).Subtract(m_imppausedts); + avgrowtime = elapsedspan.GetMilliseconds().ToLong()/(m_improwsnum + m_skiprowsnum); + stxtElapsedTime->SetLabel(elapsedspan.Format(wxT("%H:%M:%S.%l"))); + stxtAvgRowTime->SetLabel(wxString::Format(_("%d ms"), avgrowtime)); + + if (m_srcsize==-1) + { + if (incgauge) + gaugeImp->SetValue((m_improwsnum + m_skiprowsnum)%100); + } + else + { + if (m_datasrctype==TEXTFILEPAGE || m_datasrctype==ARCHFILEPAGE) + { + if (incgauge) + gaugeImp->SetValue(m_srcfile->Tell()*1000/m_srcsize); + stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(elapsedspan.GetMilliseconds().ToLong()*((m_srcsize-m_srcfile->Tell())/m_srcfile->Tell())).Format(wxT("%H:%M:%S.%l"))); + } +#if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) + else if (m_datasrctype==SRCODBCPAGE) + { + if (incgauge) + gaugeImp->SetValue(m_improwsnum*1000/m_srcsize); + stxtRemainingTime->SetLabel(wxTimeSpan::Milliseconds(avgrowtime*(m_srcsize-m_improwsnum)).Format(wxT("%H:%M:%S.%l"))); + } +#endif + } +} + bool frmImport::CheckWizardPage() { if (m_procstatus!=IMPPREPARED) @@ -3125,6 +3162,23 @@ bool frmImport::CheckWizardPage() } } + if (txtFileOptStartWith->GetValue().IsEmpty()) + { + if (m_fileoptstartwith) + { + delete m_fileoptstartwith; + m_fileoptstartwith = NULL; + } + } + else + { + if (m_fileoptstartwith) + m_fileoptstartwith->Empty(); + else + m_fileoptstartwith = new wxString; + m_fileoptstartwith->Append(txtFileOptStartWith->GetValue()); + } + if (chkFileCheckTrueFirst->GetValue()) { wxString truevals = txtFileOptTrueVals->GetValue().Trim().Trim(false); @@ -5154,46 +5208,54 @@ void frmImport::ResetWizardPage(WizardPageNumber pagenr) chcImpDest->SetSelection(0); txtDestFilePath->Disable(); btnDestFileBrowse->Disable(); + chkDestFileAppend->Disable(); } else if (impdest==IMPDESTFCOPY || impdest==IMPDESTFINS) { chcImpDest->SetSelection(1); txtDestFilePath->Enable(); btnDestFileBrowse->Enable(); + chkDestFileAppend->Enable(); } else if (impdest==IMPDESTTEST) { chcImpDest->SetSelection(2); txtDestFilePath->Disable(); btnDestFileBrowse->Disable(); + chkDestFileAppend->Disable(); } else if (impdest==IMPDESTFTEST) { chcImpDest->SetSelection(3); txtDestFilePath->Enable(); btnDestFileBrowse->Enable(); + chkDestFileAppend->Enable(); } else { chcImpDest->SetSelection(4); txtDestFilePath->Enable(); btnDestFileBrowse->Enable(); + chkDestFileAppend->Enable(); } if (rdbLogToScreen->GetValue()) { txtLogFilePath->Disable(); btnLogFileBrowse->Disable(); + chkLogFileAppend->Disable(); } else if (rdbLogToFile->GetValue()) { txtLogFilePath->Enable(); btnLogFileBrowse->Enable(); + chkLogFileAppend->Enable(); } else { txtLogFilePath->Disable(); btnLogFileBrowse->Disable(); + chkLogFileAppend->Disable(); rdbLogToScreen->SetValue(true); } @@ -5624,11 +5686,11 @@ void frmImport::ResetColSettingOnEmptyColEditor(int dbcolnr) gce_onempty->Append(IMPERRORDISP); if ((!hassrccol || impmode==IMPORTMODEINSERT) && (!m_tgtcolsisnotnull[rowidx] || (m_tgtcolsisnotnull[rowidx] && m_tgtcolshasdef[rowidx]))) gce_onempty->Append(IMPDEFAULTDISP); - if (hassrccol && !m_tgtcolsisnotnull[rowidx]) + if (!m_tgtcolsisnotnull[rowidx]) gce_onempty->Append(IMPNULLDISP); gce_onempty->Append(IMPBOOLFALSEDISP); gce_onempty->Append(IMPBOOLTRUEDISP); - if (impmode==IMPORTMODEINSERT && (m_origsrctype==TEXTFILEPAGE || m_origsrctype==ARCHFILEPAGE)) + if (impmode==IMPORTMODEINSERT) gce_onempty->Append(IMPEXPRDISP); if (onemptyact.IsEmpty() || !gce_onempty->Contains(onemptyact)) @@ -6205,10 +6267,7 @@ void frmImport::ParseColsImpSetting(int dbcolnr) int expcolnr, onemptycolnr, fmtcolnr; for (dbcolidx=loopstartnr; dbcolidxAppendText(_(" -- Please specify a constant for NOT NULL column: ") + *m_tgtcolsnames[dbcolidx]); gridColSetting->SetCellBackgroundColour(dbcolidx, expcolnr, *wxRED); + m_colsettingiserr[dbcolidx] = true; if (dbcolnr==wxNOT_FOUND || !origcolerr) - //m_colsettingerrnum++; - //else if (!origcolerr) - { - m_colsettingiserr[dbcolidx] = true; m_colsettingerrnum++; - } } else { @@ -6475,13 +6530,9 @@ void frmImport::ParseColsImpSetting(int dbcolnr) txtColsSettingLog->AppendText(_(" -- Please specify a expression for NOT NULL column: ") + *m_tgtcolsnames[dbcolidx]); gridColSetting->SetCellBackgroundColour(dbcolidx, expcolnr, *wxRED); + m_colsettingiserr[dbcolidx] = true; if (dbcolnr==wxNOT_FOUND || !origcolerr) - //m_colsettingerrnum++; - //else if (!origcolerr) - { - m_colsettingiserr[dbcolidx] = true; m_colsettingerrnum++; - } } else { @@ -6532,23 +6583,16 @@ void frmImport::ParseColsImpSetting(int dbcolnr) txtColsSettingLog->AppendText(_("Error")); txtColsSettingLog->AppendText(_(" -- Can't import any rows since setting error: ") + *m_tgtcolsnames[dbcolidx]); gridColSetting->SetCellBackgroundColour(dbcolidx, onemptycolnr, *wxRED); - + + m_colsettingiserr[dbcolidx] = true; if (dbcolnr==wxNOT_FOUND || !origcolerr) - //m_colsettingerrnum++; - //else if (!origcolerr) - { - m_colsettingiserr[dbcolidx] = true; m_colsettingerrnum++; - } } - else + if (!m_colsettingiserr[dbcolidx]) { gridColSetting->SetCellBackgroundColour(dbcolidx, onemptycolnr, *wxWHITE); if (origcolerr) - { - m_colsettingiserr[dbcolidx] = false; m_colsettingerrnum--; - } } if (impmode==IMPORTMODEINSERT && dbcolnr==wxNOT_FOUND) diff --git a/pgadmin/include/frm/frmImport.h b/pgadmin/include/frm/frmImport.h index eb3a2c9d6..36e0755c9 100755 --- a/pgadmin/include/frm/frmImport.h +++ b/pgadmin/include/frm/frmImport.h @@ -178,10 +178,8 @@ private: static void RandomSavepointName(wxChar* randname, short namelen=10); void ImportData(); - void ImportFileData(); -#if defined(__WXMSW__) || defined(__WITHIODBC__) || defined(__WITHUNIXODBC__) - void ImportODBCData(); -#endif + void RefreshProc(bool incgauge=true); + bool ImportRow(wxString& impdatalist, const int impmode, const int impdest, bool& copybolcking, bool logtoscr, bool logtofile, bool logerror, bool logwarn, bool logstmt, bool loginfo, bool logproc, @@ -212,8 +210,9 @@ private: pgCSVFile *m_srcfile; bool m_datapverr; - // match regular expression + // match regular expression & start with wxRegEx *m_fileoptimpregex; + wxString *m_fileoptstartwith; // boolean values wxArrayString *m_booltruevals; @@ -250,8 +249,8 @@ private: int m_procstatus; wxFile *m_destfile, *m_logfile; wxFileOffset m_srcsize; - wxFileOffset m_improwsnum; - int m_imperrnum, m_transrowsnum; + unsigned int m_improwsnum, m_skiprowsnum; + unsigned int m_imperrnum, m_transrowsnum; wxChar m_imptransname[16]; wxChar m_imptranssn[32]; wxDateTime m_impstartdt, m_imppausedt; diff --git a/pgadmin/include/utils/pgcsvfile.h b/pgadmin/include/utils/pgcsvfile.h index 85743aacc..3f8f0719e 100755 --- a/pgadmin/include/utils/pgcsvfile.h +++ b/pgadmin/include/utils/pgcsvfile.h @@ -60,7 +60,7 @@ public: // setting relative functions public: wxFileOffset Length() { return m_filesize; } - wxFileOffset Tell(); + wxFileOffset Tell() { return m_fileoffset; } bool Eof(); wxString GetFilePath() { return m_filepath; } @@ -156,7 +156,8 @@ public: size_t GetCurCSVLineNr() { return m_csvlinenr; } private: - void Init(); + wxFileOffset TellI(); + //void Init(); void GenHeader(); bool ValidateSetting(); @@ -189,7 +190,6 @@ private: bool m_firstrowisheader;// the first line is header bool m_backslashquote;// backslash-quote style wxChar m_quotedby; - //bool m_canpreview; size_t m_pvrowsnum; bool m_forcelinebreak; @@ -200,8 +200,9 @@ private: wxString m_filepath;// text file path wxFile *m_rawfile;// corresponding file wxFileOffset m_filesize; - //wxFileOffset m_fileoffset; - //short m_charwidth; + wxFileOffset m_baseoffset; + wxFileOffset m_fileoffset; + float m_charavglen; size_t m_linenumber;// current file line number size_t m_csvrownr; diff --git a/pgadmin/ui/frmImport.xrc b/pgadmin/ui/frmImport.xrc index 0b4a34901..d9490e475 100755 --- a/pgadmin/ui/frmImport.xrc +++ b/pgadmin/ui/frmImport.xrc @@ -1401,6 +1401,23 @@ 0 + + + wxALL + 5 + + + + + + + wxALL + 5 + + + 0 + + wxALL @@ -1945,7 +1962,7 @@ 5 1 - 2 + 3 0 0 0 @@ -1968,6 +1985,15 @@ 0 + + + wxALL + 5 + + + 0 + + @@ -2161,7 +2187,7 @@ 5 1 - 2 + 3 0 0 0 @@ -2184,6 +2210,15 @@ 0 + + + wxALL + 5 + + + 0 + + diff --git a/pgadmin/utils/pgcsvfile.cpp b/pgadmin/utils/pgcsvfile.cpp index 9f175789e..7a7459f7d 100755 --- a/pgadmin/utils/pgcsvfile.cpp +++ b/pgadmin/utils/pgcsvfile.cpp @@ -67,7 +67,10 @@ pgCSVFile::pgCSVFile() m_csvlinenr = 0; m_filesize = 0; - //m_fileoffset = 0; + m_baseoffset = 0; + m_fileoffset = 0; + m_charavglen = 0; + m_settingisok = true; //Init(); } @@ -747,7 +750,7 @@ void pgCSVFile::Close() m_ferrmsg = _("File closed."); } -wxFileOffset pgCSVFile::Tell() +wxFileOffset pgCSVFile::TellI() { #if wxUSE_STREAMS && wxUSE_ARCHIVE_STREAMS && (wxUSE_ZIPSTREAM || wxUSE_TARSTREAM) #if wxUSE_ZIPSTREAM @@ -2451,10 +2454,11 @@ bool pgCSVFile::EndPreview() size_t pvcurbufsz, readlen = 0; m_parsedoffset = 0; + size_t parsedoffset = (size_t)0; if (m_firstrowisheader) { wxFileOffset parsedrs; - size_t curcolnr = (size_t)0, parsedoffset = (size_t)0; + size_t curcolnr = (size_t)0; wxString **parsedvals; if (m_delimitedbychar) { @@ -2509,6 +2513,10 @@ bool pgCSVFile::EndPreview() if (leftbyte) memcpy(m_leftbuf+MAXWORDSZ-leftbyte, pvbufdata+m_pvdecoffset+readlen, leftbyte); + m_charavglen = TellI()*1.0/(parsedoffset+m_curfilecont.Len()); + if (parsedoffset) + m_fileoffset = (wxFileOffset)parsedoffset*m_charavglen; + ClearPreview(); return true; @@ -2536,14 +2544,21 @@ bool pgCSVFile::NextLine(wxString& linedata, wxString**& linevals, int &valsnum) wxFileOffset parsedrs; size_t curcolnr = (size_t)0, parsedoffset = m_parsedoffset, convlen; wxString **parsedvals = NULL; - char* readbuf = NULL; bool incolparsing = false, inquotedblk = false; wxString convstr, tmpstr; + char* readbuf = NULL; + size_t bufoffs = 0, readlen; + wxCSConv *txtconv = NULL; while (true) { if (!filedata[m_parsedoffset]) + { parsedrs = wxInvalidOffset; + m_curfilecont.Empty(); + m_parsedoffset = 0; + m_baseoffset = TellI(); + } else if (m_delimitedbychar) parsedvals = ParseLine(filedata, &parsedoffset, !Eof(), parsedvals, &curcolnr, &incolparsing, &inquotedblk, &parsedrs, &m_filelinenr, &linedata); @@ -2559,7 +2574,6 @@ bool pgCSVFile::NextLine(wxString& linedata, wxString**& linevals, int &valsnum) if (!readbuf) readbuf = new char[READBUFSZ]; memcpy(readbuf, m_leftbuf, MAXWORDSZ); - size_t bufoffs = 0, readlen; while (!readbuf[bufoffs] && bufoffsRead(readbuf + MAXWORDSZ, READBLKSZ); readlen = m_zipinstrm->LastRead(); + txtconv = m_entryconv; } #endif #if wxUSE_TARSTREAM @@ -2576,14 +2591,18 @@ bool pgCSVFile::NextLine(wxString& linedata, wxString**& linevals, int &valsnum) { m_tarinstrm->Read(readbuf + MAXWORDSZ, READBLKSZ); readlen = m_tarinstrm->LastRead(); + txtconv = m_entryconv; } #endif #endif if (m_rawfile) + { readlen = m_rawfile->Read(readbuf + MAXWORDSZ, READBLKSZ); + txtconv = m_fileconv; + } memset(readbuf + MAXWORDSZ + readlen, 0, READBUFSZ - MAXWORDSZ - readlen); - convlen = MB2str(m_fileconv, &convstr, readbuf, bufoffs, readlen); + convlen = MB2str(txtconv, &convstr, readbuf, bufoffs, readlen); if (convlen==wxCONV_FAILED) { m_fencisok = false; @@ -2598,12 +2617,17 @@ bool pgCSVFile::NextLine(wxString& linedata, wxString**& linevals, int &valsnum) if (m_parsedoffset) { + m_baseoffset = TellI() - (m_curfilecont.Len()-m_parsedoffset)*m_charavglen; + m_charavglen = ((m_curfilecont.Len()-m_parsedoffset)+convstr.Len())/ + ((m_curfilecont.Len()-m_parsedoffset)*m_charavglen + (MAXWORDSZ-bufoffs) + readlen); wxString tmpfilecont(m_curfilecont.GetData()+m_parsedoffset); m_curfilecont.Empty(); m_curfilecont.Append(tmpfilecont); parsedoffset -= m_parsedoffset; m_parsedoffset = 0; } + else + m_charavglen = (m_curfilecont.Len()*m_charavglen+(MAXWORDSZ-bufoffs) + readlen)/convstr.Len(); m_curfilecont.Append(convstr); filedata = m_curfilecont.GetData(); @@ -2615,6 +2639,7 @@ bool pgCSVFile::NextLine(wxString& linedata, wxString**& linevals, int &valsnum) m_prevcolvals = linevals = parsedvals; m_prevcolnum = valsnum = curcolnr+1; m_parsedoffset = parsedoffset; + m_fileoffset = m_baseoffset + (wxFileOffset)m_parsedoffset*m_charavglen; m_csvlinenr++; if (readbuf)