Differential D1726 Diff 4672 plugins/libkolab/lib/kolab_storage_cache.php

Changeset View

Standalone View

plugins/libkolab/lib/kolab_storage_cache.php

Show All 21 Lines
* along with this program. If not, see <http://www.gnu.org/licenses/>.		* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/		*/

class kolab_storage_cache		class kolab_storage_cache
{		{
const DB_DATE_FORMAT = 'Y-m-d H:i:s';		const DB_DATE_FORMAT = 'Y-m-d H:i:s';
const MAX_RECORDS = 500;		const MAX_RECORDS = 500;

public $sync_complete = false;

protected $db;		protected $db;
protected $imap;		protected $imap;
protected $folder;		protected $folder;
protected $uid2msg;		protected $uid2msg;
protected $objects;		protected $objects;
protected $metadata = array();		protected $metadata = array();
protected $folder_id;		protected $folder_id;
protected $resource_uri;		protected $resource_uri;
protected $enabled = true;		protected $enabled = true;
protected $synched = false;		protected $synched = false;
protected $synclock = false;		protected $synclock = false;
protected $ready = false;		protected $ready = false;
protected $cache_table;		protected $cache_table;
protected $cache_refresh = 3600;
protected $folders_table;		protected $folders_table;
protected $max_sql_packet;		protected $max_sql_packet;
protected $max_sync_lock_time = 600;		protected $max_sync_lock_time = 600;
protected $extra_cols = array();		protected $extra_cols = array();
protected $data_props = array();		protected $data_props = array();
protected $order_by = null;		protected $order_by = null;
protected $limit = null;		protected $limit = null;
protected $error = 0;		protected $error = 0;
Show All 26 Lines	class kolab_storage_cache
*/		*/
public function __construct(kolab_storage_folder $storage_folder = null)		public function __construct(kolab_storage_folder $storage_folder = null)
{		{
$rcmail = rcube::get_instance();		$rcmail = rcube::get_instance();
$this->db = $rcmail->get_dbh();		$this->db = $rcmail->get_dbh();
$this->imap = $rcmail->get_storage();		$this->imap = $rcmail->get_storage();
$this->enabled = $rcmail->config->get('kolab_cache', false);		$this->enabled = $rcmail->config->get('kolab_cache', false);
$this->folders_table = $this->db->table_name('kolab_folders');		$this->folders_table = $this->db->table_name('kolab_folders');
$this->cache_refresh = get_offset_sec($rcmail->config->get('kolab_cache_refresh', '12h'));
$this->server_timezone = new DateTimeZone(date_default_timezone_get());		$this->server_timezone = new DateTimeZone(date_default_timezone_get());

if ($this->enabled) {		if ($this->enabled) {
// always read folder cache and lock state from DB master		// always read folder cache and lock state from DB master
$this->db->set_table_dsn('kolab_folders', 'w');		$this->db->set_table_dsn('kolab_folders', 'w');
// remove sync-lock on script termination		// remove sync-lock on script termination
$rcmail->add_shutdown_function(array($this, '_sync_unlock'));		$rcmail->add_shutdown_function(array($this, '_sync_unlock'));
}		}
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines	class kolab_storage_cache
* Synchronize local cache data with remote		* Synchronize local cache data with remote
*/		*/
public function synchronize()		public function synchronize()
{		{
// only sync once per request cycle		// only sync once per request cycle
if ($this->synched)		if ($this->synched)
return;		return;

// increase time limit
@set_time_limit($this->max_sync_lock_time - 60);

// get effective time limit we have for synchronization (~70% of the execution time)
$time_limit = ini_get('max_execution_time') * 0.7;
$sync_start = time();

// assume sync will be completed
$this->sync_complete = true;

if (!$this->ready) {		if (!$this->ready) {
// kolab cache is disabled, synchronize IMAP mailbox cache only		// kolab cache is disabled, synchronize IMAP mailbox cache only
$this->imap_mode(true);		$this->imap_mode(true);
$this->imap->folder_sync($this->folder->name);		$this->imap->folder_sync($this->folder->name);
$this->imap_mode(false);		$this->imap_mode(false);
}		}
else {		else {
		$this->sync_start = time();

// read cached folder metadata		// read cached folder metadata
$this->_read_folder_data();		$this->_read_folder_data();

		$ctag = $this->folder->get_ctag();

		// Validate current ctag
		list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $ctag);

		if (empty($uidvalidity) \|\| empty($highestmodseq)) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (Invalid ctag)"
		), true);
		}
// check cache status ($this->metadata is set in _read_folder_data())		// check cache status ($this->metadata is set in _read_folder_data())
if ( empty($this->metadata['ctag']) \|\|		else if (
empty($this->metadata['changed']) \|\|		empty($this->metadata['ctag'])
$this->metadata['objectcount'] === null \|\|		\|\| empty($this->metadata['changed'])
$this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) \|\|		\|\| $this->metadata['ctag'] !== $ctag
$this->metadata['ctag'] != $this->folder->get_ctag() \|\|
intval($this->metadata['objectcount']) !== $this->count()
) {		) {
// lock synchronization for this folder or wait if locked		// lock synchronization for this folder or wait if locked
$this->_sync_lock();		$this->_sync_lock();

		// Run a full-sync (initial sync or continue the aborted sync)
		if (empty($this->metadata['changed']) \|\| empty($this->metadata['ctag'])) {
		$result = $this->synchronize_full();
		}
		// Synchronize only the changes since last sync
		else {
		$result = $this->synchronize_update($ctag);
		}

		// update ctag value (will be written to database in _sync_unlock())
		if ($result) {
		$this->metadata['ctag'] = $ctag;
		$this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
		}

		// remove lock
		$this->_sync_unlock();
		}
		}

		$this->check_error();
		$this->synched = time();
		}

		/**
		* Perform full cache synchronization
		*/
		protected function synchronize_full()
		{
		// get effective time limit we have for synchronization (~70% of the execution time)
		$time_limit = $this->_max_sync_lock_time() * 0.7;

		if (time() - $this->sync_start > $time_limit) {
		return false;
		}

// disable messages cache if configured to do so		// disable messages cache if configured to do so
$this->imap_mode(true);		$this->imap_mode(true);

// synchronize IMAP mailbox cache		// synchronize IMAP mailbox cache, does nothing if messages cache is disabled
$this->imap->folder_sync($this->folder->name);		$this->imap->folder_sync($this->folder->name);

// compare IMAP index with object cache index		// compare IMAP index with object cache index
$imap_index = $this->imap->index($this->folder->name, null, null, true, true);		$imap_index = $this->imap->index($this->folder->name, null, null, true, true);

$this->imap_mode(false);		$this->imap_mode(false);

		if ($imap_index->is_error()) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (SEARCH failed)"
		), true);
		return false;
		}

// determine objects to fetch or to invalidate		// determine objects to fetch or to invalidate
if (!$imap_index->is_error()) {
$imap_index = $imap_index->get();		$imap_index = $imap_index->get();
$old_index = array();
$del_index = array();		$del_index = array();
		$old_index = $this->current_index($del_index);

// read cache index		// Fetch objects and store in DB
$sql_result = $this->db->query(		$result = $this->synchronize_fetch($imap_index, $old_index, $del_index);
"SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
. " ORDER BY `msguid` DESC", $this->folder_id		if ($result) {
		// Remove redundant entries from IMAP and cache
		$rem_index = array_intersect($del_index, $imap_index);
		$del_index = array_merge(array_unique($del_index), array_diff($old_index, $imap_index));

		$this->synchronize_delete($rem_index, $del_index);
		}

		return $result;
		}

		/**
		* Perform partial cache synchronization, based on QRESYNC
		*/
		protected function synchronize_update()
		{
		if (!$this->imap->get_capability('QRESYNC')) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (no QRESYNC capability)"
		), true);

		return $this->synchronize_full();
		}

		// Handle the previous ctag
		list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $this->metadata['ctag']);

		if (empty($uidvalidity) \|\| empty($highestmodseq)) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (Invalid old ctag)"
		), true);
		return false;
		}

		// Enable QRESYNC
		$res = $this->imap->conn->enable('QRESYNC');
		if ($res === false) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (failed to enable QRESYNC/CONDSTORE)"
		), true);

		return false;
		}

		$mbox_data = $this->imap->folder_data($this->folder->name);
		if (empty($mbox_data)) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (failed to get folder state)"
		), true);

		return false;
		}

		// Check UIDVALIDITY
		if ($uidvalidity != $mbox_data['UIDVALIDITY']) {
		return $this->synchronize_full();
		}

		// QRESYNC not supported on specified mailbox
		if (!empty($mbox_data['NOMODSEQ']) \|\| empty($mbox_data['HIGHESTMODSEQ'])) {
		rcube::raise_error(array(
		'code' => 900,
		'message' => "Failed to sync the kolab cache (QRESYNC not supported on the folder)"
		), true);

		return $this->synchronize_full();
		}

		// Get modified flags and vanished messages
		// UID FETCH 1:* (FLAGS) (CHANGEDSINCE 0123456789 VANISHED)
		$result = $this->imap->conn->fetch(
		$this->folder->name, '1:*', true, array('FLAGS'), $highestmodseq, true
);		);

while ($sql_arr = $this->db->fetch_assoc($sql_result)) {		$removed = array();
// Mark all duplicates for removal (note sorting order above)		$modified = array();
// Duplicates here should not happen, but they do sometimes		$existing = $this->current_index($removed);
if (isset($old_index[$sql_arr['uid']])) {
$del_index[] = $sql_arr['msguid'];		if (!empty($result)) {
		foreach ($result as $msg) {
		$uid = $msg->uid;

		// Message marked as deleted
		if (!empty($msg->flags['DELETED'])) {
		$removed[] = $uid;
		continue;
}		}
else {
$old_index[$sql_arr['uid']] = $sql_arr['msguid'];		// Flags changed or new
		$modified[] = $uid;
}		}
}		}

// fetch new objects from imap		$new = array_diff($modified, $existing, $removed);
		$result = true;

		if (!empty($new)) {
		$result = $this->synchronize_fetch($new, $existing, $removed);

		if (!$result) {
		return false;
		}
		}

		// VANISHED found?
		$mbox_data = $this->imap->folder_data($this->folder->name);

		// Removed vanished messages from the database
		$vanished = (array) rcube_imap_generic::uncompressMessageSet($mbox_data['VANISHED']);

		// Remove redundant entries from IMAP and DB
		$vanished = array_merge($removed, array_intersect($vanished, $existing));
		$this->synchronize_delete($removed, $vanished);

		return $result;
		}

		/**
		* Fetch objects from IMAP and save into the database
		*/
		protected function synchronize_fetch($new_index, &$old_index, &$del_index)
		{
		// get effective time limit we have for synchronization (~70% of the execution time)
		$time_limit = $this->_max_sync_lock_time() * 0.7;

		if (time() - $this->sync_start > $time_limit) {
		return false;
		}

$i = 0;		$i = 0;
foreach (array_diff($imap_index, $old_index) as $msguid) {		$aborted = false;

		// fetch new objects from imap
		foreach (array_diff($new_index, $old_index) as $msguid) {
// Note: We'll store only objects matching the folder type		// Note: We'll store only objects matching the folder type
// anything else will be silently ignored		// anything else will be silently ignored
if ($object = $this->folder->read_object($msguid)) {		if ($object = $this->folder->read_object($msguid)) {
// Deduplication: remove older objects with the same UID		// Deduplication: remove older objects with the same UID
// Here we do not resolve conflicts, we just make sure		// Here we do not resolve conflicts, we just make sure
// the most recent version of the object will be used		// the most recent version of the object will be used
if ($old_msguid = $old_index[$object['uid']]) {		if ($old_msguid = $old_index[$object['uid']]) {
if ($old_msguid < $msguid) {		if ($old_msguid < $msguid) {
$del_index[] = $old_msguid;		$del_index[] = $old_msguid;
}		}
else {		else {
$del_index[] = $msguid;		$del_index[] = $msguid;
continue;		continue;
}		}
}		}

$old_index[$object['uid']] = $msguid;		$old_index[$object['uid']] = $msguid;

$this->_extended_insert($msguid, $object);		$this->_extended_insert($msguid, $object);

// check time limit and abort sync if running too long		// check time limit and abort sync if running too long
if (++$i % 50 == 0 && time() - $sync_start > $time_limit) {		if (++$i % 50 == 0 && time() - $this->sync_start > $time_limit) {
$this->sync_complete = false;		$aborted = true;
break;		break;
}		}
}		}
}		}

$this->_extended_insert(0, null);		$this->_extended_insert(0, null);

$del_index = array_unique($del_index);		return $aborted === false;
		}

// delete duplicate entries from IMAP		/**
$rem_index = array_intersect($del_index, $imap_index);		* Remove specified objects from the database and IMAP
if (!empty($rem_index)) {		*/
		protected function synchronize_delete($imap_delete, $delete)
		{
		if (!empty($imap_delete)) {
$this->imap_mode(true);		$this->imap_mode(true);
$this->imap->delete_message($rem_index, $this->folder->name);		$this->imap->delete_message($imap_delete, $this->folder->name);
$this->imap_mode(false);		$this->imap_mode(false);
}		}

// delete old/invalid entries from the cache		if (!empty($delete)) {
$del_index += array_diff($old_index, $imap_index);		$quoted_ids = join(',', array_map(array($this->db, 'quote'), $delete));
if (!empty($del_index)) {
$quoted_ids = join(',', array_map(array($this->db, 'quote'), $del_index));
$this->db->query(		$this->db->query(
"DELETE FROM `{$this->cache_table}` WHERE `folder_id` = ? AND `msguid` IN ($quoted_ids)",		"DELETE FROM `{$this->cache_table}` WHERE `folder_id` = ? AND `msguid` IN ($quoted_ids)",
$this->folder_id		$this->folder_id
);		);
}		}

// update ctag value (will be written to database in _sync_unlock())
if ($this->sync_complete) {
$this->metadata['ctag'] = $this->folder->get_ctag();
$this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
// remember the number of cache entries linked to this folder
$this->metadata['objectcount'] = $this->count();
}
}		}

// remove lock		/**
$this->_sync_unlock();		* Return current use->msguid index
		*/
		protected function current_index(&$duplicates = array())
		{
		// read cache index
		$sql_result = $this->db->query(
		"SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
		. " ORDER BY `msguid` DESC", $this->folder_id
		);

		$index = $del_index = array();

		while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
		// Mark all duplicates for removal (note sorting order above)
		// Duplicates here should not happen, but they do sometimes
		if (isset($index[$sql_arr['uid']])) {
		$duplicates[] = $sql_arr['msguid'];
}		}
		else {
		$index[$sql_arr['uid']] = $sql_arr['msguid'];
}		}

$this->check_error();
$this->synched = time();
}		}

		return $index;
		}

/**		/**
* Read a single entry from cache or from IMAP directly		* Read a single entry from cache or from IMAP directly
*		*
* @param string Related IMAP message UID		* @param string Related IMAP message UID
* @param string Object type to read		* @param string Object type to read
* @param string IMAP folder name the entry relates to		* @param string IMAP folder name the entry relates to
* @param array Hash array with object properties or null if not found		* @param array Hash array with object properties or null if not found
▲ Show 20 Lines • Show All 719 Lines • ▼ Show 20 Lines
*/		*/
protected function _read_folder_data()		protected function _read_folder_data()
{		{
// already done		// already done
if (!empty($this->folder_id) \|\| !$this->ready)		if (!empty($this->folder_id) \|\| !$this->ready)
return;		return;

$sql_arr = $this->db->fetch_assoc($this->db->query(		$sql_arr = $this->db->fetch_assoc($this->db->query(
"SELECT `folder_id`, `synclock`, `ctag`, `changed`, `objectcount`"		"SELECT `folder_id`, `synclock`, `ctag`, `changed`"
. " FROM `{$this->folders_table}` WHERE `resource` = ?",		. " FROM `{$this->folders_table}` WHERE `resource` = ?",
$this->resource_uri		$this->resource_uri
));		));

if ($sql_arr) {		if ($sql_arr) {
$this->metadata = $sql_arr;		$this->metadata = $sql_arr;
$this->folder_id = $sql_arr['folder_id'];		$this->folder_id = $sql_arr['folder_id'];
}		}
Show All 21 Lines	protected function _sync_lock()
$this->check_error();		$this->check_error();
$this->ready = false;		$this->ready = false;
return;		return;
}		}

$read_query = "SELECT `synclock`, `ctag` FROM `{$this->folders_table}` WHERE `folder_id` = ?";		$read_query = "SELECT `synclock`, `ctag` FROM `{$this->folders_table}` WHERE `folder_id` = ?";
$write_query = "UPDATE `{$this->folders_table}` SET `synclock` = ? WHERE `folder_id` = ? AND `synclock` = ?";		$write_query = "UPDATE `{$this->folders_table}` SET `synclock` = ? WHERE `folder_id` = ? AND `synclock` = ?";

		$max_lock_time = $this->_max_sync_lock_time();

// wait if locked (expire locks after 10 minutes) ...		// wait if locked (expire locks after 10 minutes) ...
// ... or if setting lock fails (another process meanwhile set it)		// ... or if setting lock fails (another process meanwhile set it)
while (		while (
(intval($this->metadata['synclock']) + $this->max_sync_lock_time > time()) \|\|		(intval($this->metadata['synclock']) + $max_lock_time > time()) \|\|
(($res = $this->db->query($write_query, time(), $this->folder_id, intval($this->metadata['synclock']))) &&		(($res = $this->db->query($write_query, time(), $this->folder_id, intval($this->metadata['synclock']))) &&
!($affected = $this->db->affected_rows($res)))		!($affected = $this->db->affected_rows($res)))
) {		) {
usleep(500000);		usleep(500000);
$this->metadata = $this->db->fetch_assoc($this->db->query($read_query, $this->folder_id));		$this->metadata = $this->db->fetch_assoc($this->db->query($read_query, $this->folder_id));
}		}

$this->synclock = $affected > 0;		$this->synclock = $affected > 0;
}		}

/**		/**
* Remove lock for this folder		* Remove lock for this folder
*/		*/
public function _sync_unlock()		public function _sync_unlock()
{		{
if (!$this->ready \|\| !$this->synclock)		if (!$this->ready \|\| !$this->synclock)
return;		return;

$this->db->query(		$this->db->query(
"UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ?, `objectcount` = ? WHERE `folder_id` = ?",		"UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ? WHERE `folder_id` = ?",
$this->metadata['ctag'],		$this->metadata['ctag'],
$this->metadata['changed'],		$this->metadata['changed'],
$this->metadata['objectcount'],
$this->folder_id		$this->folder_id
);		);

$this->synclock = false;		$this->synclock = false;
}		}

		protected function _max_sync_lock_time()
		{
		$limit = get_offset_sec(ini_get('max_execution_time'));

		if ($limit <= 0 \|\| $limit > $this->max_sync_lock_time) {
		$limit = $this->max_sync_lock_time;
		}

		return $limit;
		}

/**		/**
* Check IMAP connection error state		* Check IMAP connection error state
*/		*/
protected function check_error()		protected function check_error()
{		{
if (($err_code = $this->imap->get_error_code()) < 0) {		if (($err_code = $this->imap->get_error_code()) < 0) {
$this->error = kolab_storage::ERROR_IMAP_CONN;		$this->error = kolab_storage::ERROR_IMAP_CONN;
if (($res_code = $this->imap->get_response_code()) !== 0 && in_array($res_code, array(rcube_storage::NOPERM, rcube_storage::READONLY))) {		if (($res_code = $this->imap->get_response_code()) !== 0 && in_array($res_code, array(rcube_storage::NOPERM, rcube_storage::READONLY))) {
▲ Show 20 Lines • Show All 147 Lines • Show Last 20 Lines