Next item structure works (#5380)

* Use "LEFT JOIN" to always fetch the item. Needed for update routines.

* New conversion routine that now covers every item

* Post update is now activated

* We now use a hash based upon RIPEMD-320 for content and activity

* The hash doesn't contain the plink anymore

* Legacy item fields are now "null"able

* New hash function for a server unique item hash

* Introduction of the legacy mode (usage of old item fields)

* Code simplification

* We don't need the "uri" fields anymore in item-activity and item-content

* Use the "created" and not the "received" date for the hash

* Avoiding several notices

* Some more warnings removed

* Improved uri-hash / Likes on Diaspora are now getting a creation date

* Corrected the post update version

* Ensure an unique uri-hash

* Don't delete orhaned item data at the moment

* Partly reworked, due to strange behaviour

* Some more parts reworked

* Using the uri currently seems to be more reliable

* Using the uri here as well

* Use the hash values again

* Grouped item fields in different categories

* Notices again

* use the gravity (we always should)

* Added hint for disabled post updates

* Notices ...

* Issue #5337: Personal notes are displayed again

* Use the gravity again
This commit is contained in:
Michael Vogel 2018-07-15 20:36:20 +02:00 committed by Hypolite Petovan
parent 8ad6b65aeb
commit d3a2ed85fe
21 changed files with 315 additions and 297 deletions

View file

@ -536,26 +536,26 @@ class DBStructure
private static function FieldCommand($parameters, $create = true) {
$fieldstruct = $parameters["type"];
if (!empty($parameters["Collation"])) {
if (isset($parameters["Collation"])) {
$fieldstruct .= " COLLATE ".$parameters["Collation"];
}
if (!empty($parameters["not null"])) {
if (isset($parameters["not null"])) {
$fieldstruct .= " NOT NULL";
}
if (!empty($parameters["default"])) {
if (isset($parameters["default"])) {
if (strpos(strtolower($parameters["type"]),"int")!==false) {
$fieldstruct .= " DEFAULT ".$parameters["default"];
} else {
$fieldstruct .= " DEFAULT '".$parameters["default"]."'";
}
}
if (!empty($parameters["extra"])) {
if (isset($parameters["extra"])) {
$fieldstruct .= " ".$parameters["extra"];
}
if (!empty($parameters["comment"])) {
if (isset($parameters["comment"])) {
$fieldstruct .= " COMMENT '".dbesc($parameters["comment"])."'";
}
@ -588,11 +588,11 @@ class DBStructure
}
}
if (!empty($structure["engine"])) {
if (isset($structure["engine"])) {
$engine = " ENGINE=" . $structure["engine"];
}
if (!empty($structure["comment"])) {
if (isset($structure["comment"])) {
$comment = " COMMENT='" . dbesc($structure["comment"]) . "'";
}
@ -1158,68 +1158,76 @@ class DBStructure
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
"guid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "A unique identifier for this item"],
"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "Owner id which owns this copy of the item"],
"contact-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact.id"],
"type" => ["type" => "varchar(20)", "not null" => "1", "default" => "", "comment" => ""],
"wall" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "This item was posted to the wall of uid"],
"gravity" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
"uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
"parent" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["item" => "id"], "comment" => "item.id of the parent to this item if it is a reply of some form; otherwise this must be set to the id of this item"],
"parent-uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "uri of the parent to this item"],
"extid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"thr-parent" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri"],
"created" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Creation timestamp."],
"edited" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last edit (default is created)"],
"commented" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date of last comment/reply to this item"],
"received" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "datetime"],
"changed" => ["type" => "datetime", "not null" => "1", "default" => NULL_DATE, "comment" => "Date that something in the conversation changed, indicating clients should fetch the conversation again"],
"gravity" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"],
"owner-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Link to the contact table with uid=0 of the owner of this item"],
"owner-name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Name of the owner of this item"],
"owner-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the owner of this item"],
"owner-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the owner of this item"],
"author-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "Link to the contact table with uid=0 of the author of this item"],
"author-name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Name of the author of this item"],
"author-link" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the profile page of the author of this item"],
"author-avatar" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "Link to the avatar picture of the author of this item"],
"icid" => ["type" => "int unsigned", "relation" => ["item-content" => "id"], "comment" => "Id of the item-content table entry that contains the whole item content"],
"iaid" => ["type" => "int unsigned", "relation" => ["item-activity" => "id"], "comment" => "Id of the item-activity table entry that contains the activity data"],
"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"body" => ["type" => "mediumtext", "comment" => "item body content"],
"app" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "application which generated this item"],
"verb" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams verb"],
"object-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams object type"],
"object" => ["type" => "text", "comment" => "JSON encoded object structure unless it is an implied object (normal post)"],
"target-type" => ["type" => "varchar(100)", "not null" => "1", "default" => "", "comment" => "ActivityStreams target type if applicable (URI)"],
"target" => ["type" => "text", "comment" => "JSON encoded target structure if used"],
"postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"],
"plink" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "permalink or URL to a displayable copy of the message at its source"],
"resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"],
"event-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["event" => "id"], "comment" => "Used to link to the event.id"],
"tag" => ["type" => "mediumtext", "comment" => ""],
"attach" => ["type" => "mediumtext", "comment" => "JSON structure representing attachments to this item"],
"inform" => ["type" => "mediumtext", "comment" => ""],
"file" => ["type" => "mediumtext", "comment" => ""],
"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "text location where this item originated"],
"coord" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "longitude/latitude pair representing location where this item originated"],
"extid" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"global" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"private" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "distribution is restricted"],
"bookmark" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been bookmarked"],
"visible" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"moderated" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been deleted"],
// User specific fields. Eventually they will move to user-item
"uid" => ["type" => "mediumint unsigned", "not null" => "1", "default" => "0", "relation" => ["user" => "uid"], "comment" => "Owner id which owns this copy of the item"],
"contact-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["contact" => "id"], "comment" => "contact.id"],
"wall" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "This item was posted to the wall of uid"],
"origin" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item originated at this site"],
"pubmail" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"starred" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been favourited"],
"unseen" => ["type" => "boolean", "not null" => "1", "default" => "1", "comment" => "item has not been seen"],
"mention" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "The owner of this item was mentioned in it"],
"forum_mode" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
// User specific fields. Should possible be replaced with something different
"allow_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed contact.id '<19><78>'"],
"allow_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of allowed groups"],
"deny_cid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied contact.id"],
"deny_gid" => ["type" => "mediumtext", "comment" => "Access Control - list of denied groups"],
"private" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "distribution is restricted"],
"pubmail" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"moderated" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"visible" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"starred" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been favourited"],
"bookmark" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been bookmarked"],
"unseen" => ["type" => "boolean", "not null" => "1", "default" => "1", "comment" => "item has not been seen"],
"deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item has been deleted"],
"origin" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "item originated at this site"],
"forum_mode" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
"mention" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "The owner of this item was mentioned in it"],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => "Network from where the item comes from"],
"rendered-hash" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => ""],
"rendered-html" => ["type" => "mediumtext", "comment" => "item.body converted to html"],
"global" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
"postopts" => ["type" => "text", "comment" => "External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery"],
"inform" => ["type" => "mediumtext", "comment" => "Additional receivers of this post"],
// It is to be decided whether these fields belong to the user or the structure
"resource-id" => ["type" => "varchar(32)", "not null" => "1", "default" => "", "comment" => "Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type"],
"event-id" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "relation" => ["event" => "id"], "comment" => "Used to link to the event.id"],
// Will be replaced by the "attach" table
"attach" => ["type" => "mediumtext", "comment" => "JSON structure representing attachments to this item"],
// Seems to be only used for notes, but is filled at many places.
// Will be replaced with some general field that contain the values of "origin" and "wall" as well.
"type" => ["type" => "varchar(20)", "not null" => "1", "default" => "", "comment" => ""],
// Deprecated fields. Will be removed in upcoming versions
"file" => ["type" => "mediumtext", "comment" => "Deprecated"],
"location" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"coord" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"tag" => ["type" => "mediumtext", "comment" => "Deprecated"],
"plink" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"title" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"content-warning" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"body" => ["type" => "mediumtext", "comment" => "Deprecated"],
"app" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"verb" => ["type" => "varchar(100)", "comment" => "Deprecated"],
"object-type" => ["type" => "varchar(100)", "comment" => "Deprecated"],
"object" => ["type" => "text", "comment" => "Deprecated"],
"target-type" => ["type" => "varchar(100)", "comment" => "Deprecated"],
"target" => ["type" => "text", "comment" => "Deprecated"],
"author-name" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"author-link" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"author-avatar" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"owner-name" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"owner-link" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"owner-avatar" => ["type" => "varchar(255)", "comment" => "Deprecated"],
"rendered-hash" => ["type" => "varchar(32)", "comment" => "Deprecated"],
"rendered-html" => ["type" => "mediumtext", "comment" => "Deprecated"],
],
"indexes" => [
"PRIMARY" => ["id"],
@ -1253,8 +1261,8 @@ class DBStructure
"comment" => "Activities for items",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"uri-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 and RIPEMD-160 hash from uri"],
"uri" => ["type" => "varchar(255)", "comment" => ""],
"uri-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
"activity" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""],
],
"indexes" => [
@ -1267,8 +1275,8 @@ class DBStructure
"comment" => "Content for all posts",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "relation" => ["thread" => "iid"]],
"uri" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"uri-plink-hash" => ["type" => "char(80)", "not null" => "1", "default" => "", "comment" => "SHA-1 hash from uri and plink"],
"uri" => ["type" => "varchar(255)", "comment" => ""],
"uri-plink-hash" => ["type" => "varchar(80)", "not null" => "1", "default" => "", "comment" => "RIPEMD-128 hash from uri"],
"title" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "item title"],
"content-warning" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"body" => ["type" => "mediumtext", "comment" => "item body content"],

View file

@ -31,10 +31,7 @@ class PostUpdate
if (!self::update1206()) {
return;
}
if (!self::update1274()) {
return;
}
if (!self::update1275()) {
if (!self::update1276()) {
return;
}
}
@ -226,95 +223,62 @@ class PostUpdate
}
/**
* @brief update the "item-content" table
* @brief update the item related tables
*
* @return bool "true" when the job is done
*/
private static function update1274()
private static function update1276()
{
// Was the script completed?
if (Config::get("system", "post_update_version") >= 1274) {
if (Config::get("system", "post_update_version") >= 1276) {
return true;
}
logger("Start", LOGGER_DEBUG);
$id = Config::get("system", "post_update_version_1276_id", 0);
$fields = ['id', 'title', 'content-warning', 'body', 'location', 'tag', 'file',
'coord', 'app', 'rendered-hash', 'rendered-html', 'verb',
'object-type', 'object', 'target-type', 'target', 'plink',
'author-id', 'owner-id'];
logger("Start from item " . $id, LOGGER_DEBUG);
$condition = ["`icid` IS NULL"];
$params = ['limit' => 10000];
$items = Item::select($fields, $condition, $params);
if (!DBM::is_result($items)) {
Config::set("system", "post_update_version", 1274);
logger("Done", LOGGER_DEBUG);
return true;
}
$fields = array_merge(Item::MIXED_CONTENT_FIELDLIST, ['network', 'author-id', 'owner-id', 'tag', 'file',
'author-name', 'author-avatar', 'author-link', 'owner-name', 'owner-avatar', 'owner-link', 'id']);
$start_id = $id;
$rows = 0;
$condition = ["`id` > ?", $id];
$params = ['order' => ['id'], 'limit' => 10000];
$items = Item::select($fields, $condition, $params);
while ($item = Item::fetch($items)) {
// Clearing the author and owner data if there is an id.
if ($item['author-id'] > 0) {
$item['author-name'] = '';
$item['author-link'] = '';
$item['author-avatar'] = '';
$id = $item['id'];
if (empty($item['author-id'])) {
$default = ['url' => $item['author-link'], 'name' => $item['author-name'],
'photo' => $item['author-avatar'], 'network' => $item['network']];
$item['author-id'] = Contact::getIdForURL($item["author-link"], 0, false, $default);
}
if ($item['owner-id'] > 0) {
$item['owner-name'] = '';
$item['owner-link'] = '';
$item['owner-avatar'] = '';
if (empty($item['owner-id'])) {
$default = ['url' => $item['owner-link'], 'name' => $item['owner-name'],
'photo' => $item['owner-avatar'], 'network' => $item['network']];
$item['owner-id'] = Contact::getIdForURL($item["owner-link"], 0, false, $default);
}
Item::update($item, ['id' => $item['id']]);
Item::update($item, ['id' => $id]);
++$rows;
}
dba::close($items);
logger("Processed rows: " . $rows, LOGGER_DEBUG);
return true;
}
/**
* @brief update the "item-activity" table
*
* @return bool "true" when the job is done
*/
private static function update1275()
{
// Was the script completed?
if (Config::get("system", "post_update_version") >= 1275) {
return true;
}
Config::set("system", "post_update_version_1276_id", $id);
logger("Start", LOGGER_DEBUG);
logger("Processed rows: " . $rows . " - last processed item: " . $id, LOGGER_DEBUG);
$fields = ['id', 'verb'];
$condition = ["`iaid` IS NULL AND NOT `icid` IS NULL AND `verb` IN (?, ?, ?, ?, ?)",
ACTIVITY_LIKE, ACTIVITY_DISLIKE, ACTIVITY_ATTEND, ACTIVITY_ATTENDNO, ACTIVITY_ATTENDMAYBE];
$params = ['limit' => 10000];
$items = Item::select($fields, $condition, $params);
if (!DBM::is_result($items)) {
Config::set("system", "post_update_version", 1275);
if ($start_id == $id) {
Config::set("system", "post_update_version", 1276);
logger("Done", LOGGER_DEBUG);
return true;
}
$rows = 0;
while ($item = Item::fetch($items)) {
Item::update($item, ['id' => $item['id']]);
++$rows;
}
dba::close($items);
logger("Processed rows: " . $rows, LOGGER_DEBUG);
return true;
return false;
}
}