Vekseid
Active member
I only rewrote the PrivateMessages step, naturally. There are other problems with the importer but nothing so drastic.
I did this because the standard importer creates an utterly unmanageable mess when tens of millions of PMs are involved. Currently running through a full test import.
Comments and criticism welcome. It is a bit hackish as I'm not planning on doing this again.
I did this because the standard importer creates an utterly unmanageable mess when tens of millions of PMs are involved. Currently running through a full test import.
Comments and criticism welcome. It is a bit hackish as I'm not planning on doing this again.
Code:
public function stepPrivateMessages($start, array $options)
{
$options = array_merge(array(
'current' => 1,
'max' => false
), $options);
$sDb = $this->_sourceDb;
$prefix = $this->_prefix;
/* @var $model XenForo_Model_Import */
$model = $this->_importModel;
$next = $start;
if ($options['max'] === false)
{
$options['max'] = $sDb->fetchOne('
SELECT MAX(uid)
FROM ' . $prefix . 'users
');
}
$startName = $sDb->fetchOne('
SELECT username
FROM ' . $prefix . 'users
WHERE uid = ' . $sDb->quote($start) . '
');
$currName = $sDb->fetchOne('
SELECT username
FROM ' . $prefix . 'users
WHERE uid = ' . $sDb->quote($options['current']) . '
');
$pms = $sDb->fetchAll(
'
SELECT pms.*
FROM ' . $prefix . 'privatemessages AS pms
WHERE (pms.toid = ' . $sDb->quote($start) . ' AND pms.fromid = ' . $sDb->quote($options['current']) . ')
OR (pms.fromid = ' . $sDb->quote($start) . ' AND pms.toid = ' . $sDb->quote($options['current']) . ')
ORDER BY pms.pmid
'
);
if ($pms) {
$userIds = array();
foreach ($pms AS $pm) {
$userIds[$pm['toid']] = $pm['toid'];
$userIds[$pm['fromid']] = $pm['fromid'];
}
$mapUserIds = $model->getImportContentMap('user', $userIds);
$startUserId = $this->_mapLookUp($mapUserIds, $start);
$currUserId = $this->_mapLookUp($mapUserIds, $options['current']);
if ($startUserId || $currUserId) {
$recipients = [];
$startIsFrom = false;
if ($start === $pms[0]['fromid']) {
$startIsFrom = true;
}
$lastpm = count($pms) - 1;
$startRead = (int) $pms[$lastpm]['readtime'];
$currRead = (int) $pms[$lastpm]['dateline'];
$startLast = false;
if ($start === $pms[$lastpm]['fromid']) {
$startLast = true;
$startRead = (int) $pms[$lastpm]['dateline'];
$currRead = (int) $pms[$lastpm]['readtime'];
}
if ($lastpm === 1) {
if ($startIsFrom) {
$startRead = (int) max($startRead, $pms[0]['dateline']);
$currRead = (int) max($currRead, $pms[0]['readtime']);
}
else {
$startRead = (int) max($startRead, $pms[0]['readtime']);
$currRead = (int) max($currRead, $pms[0]['dateline']);
}
}
else if ($lastpm && $startLast) {
$currDateline = $sDb->fetchOne('
SELECT max(dateline)
FROM ' . $prefix . 'privatemessages
WHERE (fromid = ' . $sDb->quote($start) . ' AND toid = ' . $sDb->quote($options['current']) . ')
');
$currReadtime = $sDb->fetchOne('
SELECT max(readtime)
FROM ' . $prefix . 'privatemessages
WHERE (fromid = ' . $sDb->quote($start) . ' AND toid = ' . $sDb->quote($options['current']) . ')
');
$currRead = (int) max($currReadtime, $currDateline);
}
else if ($lastpm) {
$startDateline = $sDb->fetchOne('
SELECT max(dateline)
FROM ' . $prefix . 'privatemessages
WHERE (toid = ' . $sDb->quote($start) . ' AND fromid = ' . $sDb->quote($options['current']) . ')
');
$startReadtime = $sDb->fetchOne('
SELECT max(readtime)
FROM ' . $prefix . 'privatemessages
WHERE (toid = ' . $sDb->quote($start) . ' AND fromid = ' . $sDb->quote($options['current']) . ')
');
$startRead = (int) max($startDateline, $startReadtime);
}
if ($startUserId) {
$recipients[$startUserId] = array(
'username' => $this->_convertToUtf8($startName, true),
'last_read_date' => $startRead,
'recipient_state' => 'active'
);
}
else {
$startUserId = 0;
}
if ($currUserId && $currUserId !== $startUserId) {
$recipients[$currUserId] = array(
'username' => $this->_convertToUtf8($currName, true),
'last_read_date' => $currRead,
'recipient_state' => 'active'
);
}
$convoSubject = $pms[0]['subject'];
if (substr($convoSubject, 0, 4) === 'Re: ') {
$convoSubject = substr($convoSubject, 4);
}
$conversation = array(
'title' => $this->_convertToUtf8($convoSubject, true),
'start_date' => $pms[0]['dateline'],
'open_invite' => 0,
'conversation_open' => 1
);
if ($startIsFrom) {
$conversation['user_id'] = $startUserId;
$conversation['username'] = $this->_convertToUtf8($startName, true);
}
else {
$conversation['user_id'] = $currUserId;
$conversation['username'] = $this->_convertToUtf8($currName, true);
}
$currentText = '';
$messages = [];
foreach ($pms as $pm) {
$pmUserId = $this->_mapLookUp($mapUserIds, $pm['fromid']);
if (!$pmUserId) {
$pmUserId = 0;
}
$pmName = $currName;
if ($pmUserId === $startUserId) {
$pmName = $startName;
}
if ($currentText === $pm['message']) {
continue;
}
else {
$currentText = $pm['message'];
}
$messages[$pm['pmid']] = array(
'message_date' => $pm['dateline'],
'user_id' => $pmUserId,
'username' => $this->_convertToUtf8($pmName),
'message' => $this->_sanitizeBbCode($pm['message'])
);
}
XenForo_Db::beginTransaction();
$model->importConversation($pms[0]['pmid'], $conversation, $recipients, $messages);
XenForo_Db::commit();
}
}
$nFrom = $startName = $sDb->fetchOne('
SELECT min(fromid)
FROM ' . $prefix . 'privatemessages
WHERE (toid = ' . $sDb->quote($start) . ' AND fromid > ' . $sDb->quote($options['current']) . ')
');
$nTo = $startName = $sDb->fetchOne('
SELECT min(toid)
FROM ' . $prefix . 'privatemessages
WHERE (fromid = ' . $sDb->quote($start) . ' AND toid > ' . $sDb->quote($options['current']) . ')
');
$oldcurrent = $options['current'];
if (!$nTo && !$nFrom) {
$this->_session->incrementStepImportTotal(1);
$next += 1;
$options['current'] = $next;
}
else if ($nTo) {
$options['current'] = $nTo;
}
else if ($nFrom) {
$options['current'] = $nFrom;
}
else {
$options['current'] = (int) min($nTo, $nFrom);
}
return array($next, $options, $this->_getProgressOutput($next, $options['max']));
}