)]}' {"/COMMIT_MSG":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bb4e6b2061f51a5cf0ccfc61a3b43d17dfeb09d8","unresolved":true,"context_lines":[{"line_number":18,"context_line":"reconciler on the next container-replication run as it enqueues objects"},{"line_number":19,"context_line":"to reconcile."},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"This is the first version and is based on clays migration work. And"},{"line_number":22,"context_line":"seems to be working when testing on my SAIO."},{"line_number":23,"context_line":"But we might decide we don\u0027t want it based on Clay\u0027s parent patch so"},{"line_number":24,"context_line":"might have to strip it back and just update the policy_stat table"},{"line_number":25,"context_line":"directly to change the storage_policy."},{"line_number":26,"context_line":""},{"line_number":27,"context_line":"Change-Id: Ib1d6197caa15d0974334be3d7eff24d4342ac378"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"fe88350f_e4ad8aff","line":25,"range":{"start_line":21,"start_character":0,"end_line":25,"end_character":38},"updated":"2021-07-22 11:08:55.000000000","message":"needs updating since rebase","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"}],"swift/container/backend.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":1009,"context_line":" return False"},{"line_number":1010,"context_line":""},{"line_number":1011,"context_line":" def migrate_to_storage_policy(self, policy_index, timestamp):"},{"line_number":1012,"context_line":" if policy_index \u003d\u003d self.storage_policy_index:"},{"line_number":1013,"context_line":" return False"},{"line_number":1014,"context_line":" put_offset, del_offset \u003d (1, 2) if self.is_deleted() else (2, 1)"},{"line_number":1015,"context_line":" put_timestamp \u003d Timestamp(timestamp, offset\u003dput_offset)"}],"source_content_type":"text/x-python","patch_set":18,"id":"7e3c52c9_0d079bd1","line":1012,"range":{"start_line":1012,"start_character":27,"end_line":1012,"end_character":52},"updated":"2021-08-20 14:10:59.000000000","message":"self.storage_policy_index returns a cached so this check only works on the first call after a get_info()","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":1032,"context_line":" put_timestamp.internal,"},{"line_number":1033,"context_line":" delete_timestamp.internal,"},{"line_number":1034,"context_line":" status_changed_at_timestamp.internal))"},{"line_number":1035,"context_line":" conn.commit()"},{"line_number":1036,"context_line":" return True"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":" def set_storage_policy_index(self, policy_index, timestamp\u003dNone):"}],"source_content_type":"text/x-python","patch_set":18,"id":"38315a24_641fb6b1","line":1035,"updated":"2021-08-20 14:10:59.000000000","message":"we should reset self._storage_policy_index here","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a56353c40e5d9cf0ace223e67f3d3a9baf17247a","unresolved":true,"context_lines":[{"line_number":1035,"context_line":" conn.commit()"},{"line_number":1036,"context_line":" return True"},{"line_number":1037,"context_line":""},{"line_number":1038,"context_line":" def set_storage_policy_index(self, policy_index, timestamp\u003dNone):"},{"line_number":1039,"context_line":" \"\"\""},{"line_number":1040,"context_line":" Update the container_stat policy_index and status_changed_at."},{"line_number":1041,"context_line":" \"\"\""}],"source_content_type":"text/x-python","patch_set":18,"id":"0a732c49_84d8f0c0","line":1038,"updated":"2021-08-20 18:41:56.000000000","message":"why isn\u0027t this method sufficient? how are they even different?!","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"}],"swift/container/sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6dbdc63e6be3ada928d8c5db262528b0eeab127a","unresolved":true,"context_lines":[{"line_number":1069,"context_line":""},{"line_number":1070,"context_line":" def _update_storage_policy_index(self, root_resp, broker):"},{"line_number":1071,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1072,"context_line":" if root_sp_index !\u003d broker.get_info()[\u0027storage_policy_index\u0027]:"},{"line_number":1073,"context_line":" root_put_ts \u003d root_resp.headers.get(\u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":1074,"context_line":" if root_put_ts:"},{"line_number":1075,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""}],"source_content_type":"text/x-python","patch_set":1,"id":"ca250987_2a151974","line":1072,"range":{"start_line":1072,"start_character":11,"end_line":1072,"end_character":24},"updated":"2021-07-14 08:50:27.000000000","message":"paranoia: let\u0027s check it\u0027s not None!","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b803f2c6e128537763509deb0316867d2c23a08","unresolved":true,"context_lines":[{"line_number":1069,"context_line":""},{"line_number":1070,"context_line":" def _update_storage_policy_index(self, root_resp, broker):"},{"line_number":1071,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1072,"context_line":" if root_sp_index !\u003d broker.get_info()[\u0027storage_policy_index\u0027]:"},{"line_number":1073,"context_line":" root_put_ts \u003d root_resp.headers.get(\u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":1074,"context_line":" if root_put_ts:"},{"line_number":1075,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""}],"source_content_type":"text/x-python","patch_set":1,"id":"2c94de92_fec7e2a9","line":1072,"range":{"start_line":1072,"start_character":11,"end_line":1072,"end_character":24},"in_reply_to":"ca250987_2a151974","updated":"2021-07-19 01:58:56.000000000","message":"kk, will fix.","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6dbdc63e6be3ada928d8c5db262528b0eeab127a","unresolved":true,"context_lines":[{"line_number":1074,"context_line":" if root_put_ts:"},{"line_number":1075,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1076,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1077,"context_line":" broker.migrate_to_storage_policy(root_sp_index, root_put_ts)"},{"line_number":1078,"context_line":""},{"line_number":1079,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1080,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"d118d28b_8e2cdabc","line":1077,"range":{"start_line":1077,"start_character":23,"end_line":1077,"end_character":48},"updated":"2021-07-14 08:50:27.000000000","message":"does this cope with repeated migrations? I see migrate_to_storage_policy bumps the timestamps, but just want to be sure that the bumped timestamp is what we would then be passing in on a repeat migration - I *think* it is i.e. X-Backend-Put-Timestamp has the offset part, correct??\n\nso for put time of T, first migration takes us to put time of T_2 and next migration would take us to T_4 etc","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b803f2c6e128537763509deb0316867d2c23a08","unresolved":true,"context_lines":[{"line_number":1074,"context_line":" if root_put_ts:"},{"line_number":1075,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1076,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1077,"context_line":" broker.migrate_to_storage_policy(root_sp_index, root_put_ts)"},{"line_number":1078,"context_line":""},{"line_number":1079,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1080,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":1,"id":"5ed45578_1e56622f","line":1077,"range":{"start_line":1077,"start_character":23,"end_line":1077,"end_character":48},"in_reply_to":"d118d28b_8e2cdabc","updated":"2021-07-19 01:58:56.000000000","message":"yeah, X-Backend-Put-Timestamp is an internal scoped timestamp and includes the offset. So if you run it again the offset continues to increment, in fact becaus the root has been migrated (so it gets 1 and 2 offset) the shard becomes 3 and 4:\n\nPath: /.shards_AUTH_admin/shardme-7820d6e9550a1661e01e0538aea8cc1b-1626070126.04615-0\n Account: .shards_AUTH_admin\n Container: shardme-7820d6e9550a1661e01e0538aea8cc1b-1626070126.04615-0\n Deleted: False\n Container Hash: bad66758a662065b6167ecf36df4b34b\nMetadata:\n Created at: 2021-07-12T06:09:33.247860 (1626070173.24786)\n Put Timestamp: 2021-07-13T07:06:17.728800 (1626159977.72880_0000000000000004)\n Delete Timestamp: 2021-07-13T07:06:17.728800 (1626159977.72880_0000000000000003)\n Status Timestamp: 2021-07-13T07:06:17.728800 (1626159977.72880_0000000000000004)\n Object Count: 0\n Bytes Used: 0\n Storage Policy: ec62 (1)\n Reported Put Timestamp: 1970-01-01T00:00:00.000000 (0)\n Reported Delete Timestamp: 1970-01-01T00:00:00.000000 (0)\n Reported Object Count: 0\n Reported Bytes Used: 0\n Chexor: 8120931e62a8f648201fac11827b4112\n UUID: f8bf094f-11cc-4718-994b-fedb64c5e37e\n X-Container-Sync-Point2: -1\n X-Container-Sync-Point1: -1\n System Metadata: {\u0027X-Container-Sysmeta-Sharding\u0027: \u0027True\u0027, \u0027X-Container-Sysmeta-Shard-Quoted-Root\u0027: \u0027AUTH_admin/shardme\u0027}\nNo user metadata found in db file\nSharding Metadata:\n Type: shard\n State: unsharded\nPartition 747\nHash bad66758a662065b6167ecf36df4b34b","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"6dbdc63e6be3ada928d8c5db262528b0eeab127a","unresolved":true,"context_lines":[{"line_number":1111,"context_line":" # only match on name."},{"line_number":1112,"context_line":" if shard_range.name \u003d\u003d own_shard_range.name:"},{"line_number":1113,"context_line":" own_shard_range_from_root \u003d shard_range"},{"line_number":1114,"context_line":" # Only potentually update storage_policy_index if"},{"line_number":1115,"context_line":" # we find the SR with the root."},{"line_number":1116,"context_line":" self._update_storage_policy_index(root_resp, broker)"},{"line_number":1117,"context_line":" break"}],"source_content_type":"text/x-python","patch_set":1,"id":"73966abf_6bcd0b1e","line":1114,"range":{"start_line":1114,"start_character":31,"end_line":1114,"end_character":42},"updated":"2021-07-14 08:50:27.000000000","message":"nit s/potentually/potentially/","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"2b803f2c6e128537763509deb0316867d2c23a08","unresolved":false,"context_lines":[{"line_number":1111,"context_line":" # only match on name."},{"line_number":1112,"context_line":" if shard_range.name \u003d\u003d own_shard_range.name:"},{"line_number":1113,"context_line":" own_shard_range_from_root \u003d shard_range"},{"line_number":1114,"context_line":" # Only potentually update storage_policy_index if"},{"line_number":1115,"context_line":" # we find the SR with the root."},{"line_number":1116,"context_line":" self._update_storage_policy_index(root_resp, broker)"},{"line_number":1117,"context_line":" break"}],"source_content_type":"text/x-python","patch_set":1,"id":"5c4e2b9e_97216102","line":1114,"range":{"start_line":1114,"start_character":31,"end_line":1114,"end_character":42},"in_reply_to":"73966abf_6bcd0b1e","updated":"2021-07-19 01:58:56.000000000","message":"Ack","commit_id":"94f5fe2955849fce65115caefdff083953581d1e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bb4e6b2061f51a5cf0ccfc61a3b43d17dfeb09d8","unresolved":true,"context_lines":[{"line_number":1067,"context_line":" self._increment_stat(\u0027audit_root\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":1068,"context_line":" return True"},{"line_number":1069,"context_line":""},{"line_number":1070,"context_line":" def _update_storage_policy_index(self, root_resp, broker):"},{"line_number":1071,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1072,"context_line":" if root_sp_index is not None and \\"},{"line_number":1073,"context_line":" root_sp_index !\u003d broker.get_info()[\u0027storage_policy_index\u0027]:"}],"source_content_type":"text/x-python","patch_set":3,"id":"f23b3f7d_a973f311","line":1070,"range":{"start_line":1070,"start_character":4,"end_line":1070,"end_character":61},"updated":"2021-07-22 11:08:55.000000000","message":"this deviates from the otherwise fairly consistent pattern of broker being the *first* arg when broker is an arg in the class\u0027s methods, so for consistency I think it\u0027s worth switching to\n\n def _update_storage_policy_index(self, broker, root_resp)","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e8186b811739a67eb688773cd8b451b15c34057f","unresolved":false,"context_lines":[{"line_number":1067,"context_line":" self._increment_stat(\u0027audit_root\u0027, \u0027success\u0027, statsd\u003dTrue)"},{"line_number":1068,"context_line":" return True"},{"line_number":1069,"context_line":""},{"line_number":1070,"context_line":" def _update_storage_policy_index(self, root_resp, broker):"},{"line_number":1071,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1072,"context_line":" if root_sp_index is not None and \\"},{"line_number":1073,"context_line":" root_sp_index !\u003d broker.get_info()[\u0027storage_policy_index\u0027]:"}],"source_content_type":"text/x-python","patch_set":3,"id":"75b4e2e7_564f5029","line":1070,"range":{"start_line":1070,"start_character":4,"end_line":1070,"end_character":61},"in_reply_to":"f23b3f7d_a973f311","updated":"2021-07-23 03:07:21.000000000","message":"Yup, let\u0027s keep the patten. Will fix.","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bb4e6b2061f51a5cf0ccfc61a3b43d17dfeb09d8","unresolved":true,"context_lines":[{"line_number":1075,"context_line":" if root_put_ts:"},{"line_number":1076,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1077,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1078,"context_line":" broker.migrate_to_storage_policy(root_sp_index, root_put_ts)"},{"line_number":1079,"context_line":""},{"line_number":1080,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1081,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"d0e0d174_17335daa","line":1078,"range":{"start_line":1078,"start_character":64,"end_line":1078,"end_character":75},"updated":"2021-07-22 11:08:55.000000000","message":"do we want the shard put timestamp to move to the root put timestamp? assuming the root put timestamp is older than the shard\u0027s, then what happens when the shard next replicates with a peer that has not updated its policy? won\u0027t the peer, with a newer put_ts, win and the policy will revert?\n\nAlso, what happens if on next audit cycle the shard gets a different policy from a different root container replica? does it just accept that as the new truth? I don\u0027t see any check in migrate_to_storage_policy that the root_put_ts is newer than the current one?","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e8186b811739a67eb688773cd8b451b15c34057f","unresolved":true,"context_lines":[{"line_number":1075,"context_line":" if root_put_ts:"},{"line_number":1076,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1077,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1078,"context_line":" broker.migrate_to_storage_policy(root_sp_index, root_put_ts)"},{"line_number":1079,"context_line":""},{"line_number":1080,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1081,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"51a93b75_bedd3590","line":1078,"range":{"start_line":1078,"start_character":64,"end_line":1078,"end_character":75},"in_reply_to":"d0e0d174_17335daa","updated":"2021-07-23 03:07:21.000000000","message":"hmm, yeah, new timestamps could make things messy. Need to think about this.\n\nMight push up a probe test and then come back to this.","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d5ecf85c03406c40b5a2442ca0471d77ce7a6dc6","unresolved":true,"context_lines":[{"line_number":1091,"context_line":" if shard_sp_timestamp and shard_sp_timestamp \u003e\u003d root_sp_timestamp:"},{"line_number":1092,"context_line":" # local policy_change timestamp as newer then roots so ignore."},{"line_number":1093,"context_line":" return"},{"line_number":1094,"context_line":" if root_sp_index is not None and \\"},{"line_number":1095,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1096,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1097,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"}],"source_content_type":"text/x-python","patch_set":7,"id":"5abb4a60_4ba89249","line":1094,"range":{"start_line":1094,"start_character":11,"end_line":1094,"end_character":36},"updated":"2021-07-27 00:15:44.000000000","message":"This check is probably not required now and we know it can\u0027t be None.","commit_id":"5b6648951ba637eafcf4fcda47df89acdb5b9c20"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4962a3cabeb2f5c57f16fc9d88e461b51c2e58ad","unresolved":true,"context_lines":[{"line_number":1082,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"}],"source_content_type":"text/x-python","patch_set":8,"id":"e2690068_f78f2286","line":1085,"range":{"start_line":1085,"start_character":12,"end_line":1085,"end_character":47},"updated":"2021-07-28 14:24:28.000000000","message":"this a new header, but where is it generated?\n\ncan\u0027t this timestamp be the root\u0027s put-timestamp or status-changed-at?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce97765f208019a9c380ffe86e4e9d707a2f7820","unresolved":true,"context_lines":[{"line_number":1082,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"}],"source_content_type":"text/x-python","patch_set":8,"id":"cf0dfc19_90a3bada","line":1085,"range":{"start_line":1085,"start_character":12,"end_line":1085,"end_character":47},"in_reply_to":"10bd1bcb_c379e800","updated":"2021-07-29 17:57:22.000000000","message":"I think we have to consider this patch on its own. The problem I thought we were fixing here is that a root policy might change, on master today, either via split brain or a deleted container that is revived with a different policy.\n\nSo this has to work with existing headers IMHO.\n\nI think that does mean there is potential for a flip-flop.\n\nThe flip flop could be avoided if the shards were created with the policy metadata having same timestamp as the root, but that\u0027s hard.\n\nHowever, the root could set X-Container-Sysmeta-Policy-Change when it creates the shard, so that would then avoid the flip-flop I think?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d81b38a3fa8f3cbc871632097464d4d118cbabec","unresolved":true,"context_lines":[{"line_number":1082,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"}],"source_content_type":"text/x-python","patch_set":8,"id":"10bd1bcb_c379e800","line":1085,"range":{"start_line":1085,"start_character":12,"end_line":1085,"end_character":47},"in_reply_to":"e2690068_f78f2286","updated":"2021-07-29 00:43:40.000000000","message":"Well it gets applied in the follow up patch when we change the policy in the container server (so for this patch it might have be manual. But now thinking of it, falling back to roots put_timestamp might be ok. Now that we can track the latest timestamp in the shard as X-Container-Sysmeta-Policy-Change.\n\nSomehow I convinced my self I had to be careful of split brain and stale roots.. but maybe not so much.\n\nif we have a shard with a newer put timestamp but it doesn\u0027t have a Policy-Change yet and the root it\u0027s talking to doesn\u0027t have one either but has a different policy do we change?\nMaybe we should.. it\u0027ll get a policy-change as the root put_timestamp. If something changes (via the follow up patch) or there is a newer root out there, then it\u0027s put timestamp would bring it back to correctness. \nSo the question then becomes, do we allow this flappy edge-case (knowing it\u0027ll only happen once). If so we then get automatic shard migration which covers a split-brain difference in policies.. but would these really be sharded?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4962a3cabeb2f5c57f16fc9d88e461b51c2e58ad","unresolved":true,"context_lines":[{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"},{"line_number":1089,"context_line":" return"}],"source_content_type":"text/x-python","patch_set":8,"id":"7c2ced5e_0cf20318","line":1086,"range":{"start_line":1086,"start_character":8,"end_line":1086,"end_character":26},"updated":"2021-07-28 14:24:28.000000000","message":"var name is shard_sp_timestamp, but this is actually the value we learnt from a root timestamp?? The shard\u0027s policy has a different timestamp correct?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d81b38a3fa8f3cbc871632097464d4d118cbabec","unresolved":true,"context_lines":[{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"},{"line_number":1089,"context_line":" return"}],"source_content_type":"text/x-python","patch_set":8,"id":"c1b03491_c607b59c","line":1086,"range":{"start_line":1086,"start_character":8,"end_line":1086,"end_character":26},"in_reply_to":"7c2ced5e_0cf20318","updated":"2021-07-29 00:43:40.000000000","message":"no it\u0027s coming from the shard broker right, so it\u0027s the shards current view of the timestamp (or none if it hasn\u0027t got one yet).\n\nWe grab the policy index and policy-change from the root resp. And compare it with the index and policy change in the shard broker.","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce97765f208019a9c380ffe86e4e9d707a2f7820","unresolved":true,"context_lines":[{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027)"},{"line_number":1086,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1087,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"},{"line_number":1088,"context_line":" if root_sp_timestamp is None:"},{"line_number":1089,"context_line":" return"}],"source_content_type":"text/x-python","patch_set":8,"id":"af1d5ee4_88f489be","line":1086,"range":{"start_line":1086,"start_character":8,"end_line":1086,"end_character":26},"in_reply_to":"c1b03491_c607b59c","updated":"2021-07-29 17:57:22.000000000","message":"yeah, but I found the name confusing because it is the shard\u0027s copy of the root policy timestamp, it is not the \u0027shard storage policy timestamp\u0027","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4962a3cabeb2f5c57f16fc9d88e461b51c2e58ad","unresolved":true,"context_lines":[{"line_number":1095,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1096,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1097,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1098,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"},{"line_number":1099,"context_line":" broker.update_metadata("},{"line_number":1100,"context_line":" {\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":1101,"context_line":" (root_sp_timestamp.internal, Timestamp.now().internal)})"},{"line_number":1102,"context_line":""},{"line_number":1103,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1104,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"579c04cc_351b7a78","line":1101,"range":{"start_line":1098,"start_character":12,"end_line":1101,"end_character":73},"updated":"2021-07-28 14:24:28.000000000","message":"so IIUC, we have to use Timestamp.now() when migrating the shard\u0027s policy because otherwise the shard\u0027s old policy meta-timestamp will win and the root policy won\u0027t merge, correct?\n\n...and then we write down the basis upon which we decided to force the shard\u0027s policy to change, to govern any future update?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"d81b38a3fa8f3cbc871632097464d4d118cbabec","unresolved":true,"context_lines":[{"line_number":1095,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1096,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1097,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1098,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"},{"line_number":1099,"context_line":" broker.update_metadata("},{"line_number":1100,"context_line":" {\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":1101,"context_line":" (root_sp_timestamp.internal, Timestamp.now().internal)})"},{"line_number":1102,"context_line":""},{"line_number":1103,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1104,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"b7685199_3dff4242","line":1101,"range":{"start_line":1098,"start_character":12,"end_line":1101,"end_character":73},"in_reply_to":"579c04cc_351b7a78","updated":"2021-07-29 00:43:40.000000000","message":"Yeah, There is a chance the put timestamp of the root is older then the shard. So we can\u0027t use it.. but was still wanting to store the roots put ts (as it was tracking the time it changed policy).\n\nDoes that make sense? or should be set the shards policy-change to Timestamp.now() too, becuase if there is another change to root that it\u0027ll get a new put_timestamp.. but this goes against what I said to respond on line 1085. If there is an old root and we flap over to it, then we\u0027d never flap back to a slightly newer root. So tracking the roots put_ts does seem to make sense.","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce97765f208019a9c380ffe86e4e9d707a2f7820","unresolved":true,"context_lines":[{"line_number":1095,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1096,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1097,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1098,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"},{"line_number":1099,"context_line":" broker.update_metadata("},{"line_number":1100,"context_line":" {\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":1101,"context_line":" (root_sp_timestamp.internal, Timestamp.now().internal)})"},{"line_number":1102,"context_line":""},{"line_number":1103,"context_line":" def _audit_shard_container(self, broker):"},{"line_number":1104,"context_line":" self._increment_stat(\u0027audit_shard\u0027, \u0027attempted\u0027)"}],"source_content_type":"text/x-python","patch_set":8,"id":"c4392c88_b43bbc03","line":1101,"range":{"start_line":1098,"start_character":12,"end_line":1101,"end_character":73},"in_reply_to":"b7685199_3dff4242","updated":"2021-07-29 17:57:22.000000000","message":"Yes, I agree we cannot use the root put timestamp as the meta timestamp for the shard policy index because the shard\u0027s db is likely to have a newer meta timestamp, so the update would fail.\n\nBut we do need to stash the root timestamp, so that we only ever change the shard policy if the root\u0027s timestamp is newer than the one we have stashed.","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce97765f208019a9c380ffe86e4e9d707a2f7820","unresolved":true,"context_lines":[{"line_number":1082,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027,"},{"line_number":1086,"context_line":" root_resp.headers.get(\u0027X-Backend-Put-Timestamp\u0027))"},{"line_number":1087,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1088,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"}],"source_content_type":"text/x-python","patch_set":9,"id":"0114c1e8_80d6e0ab","line":1085,"range":{"start_line":1085,"start_character":12,"end_line":1085,"end_character":47},"updated":"2021-07-29 17:57:22.000000000","message":"I don\u0027t think this header should be referred to until something creates it.\n\nHowever (see other comment) maybe the sharder should be sending it when it creates shard containers?","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":true,"context_lines":[{"line_number":1082,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1083,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1084,"context_line":" root_sp_timestamp \u003d root_resp.headers.get("},{"line_number":1085,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027,"},{"line_number":1086,"context_line":" root_resp.headers.get(\u0027X-Backend-Put-Timestamp\u0027))"},{"line_number":1087,"context_line":" shard_sp_timestamp \u003d Timestamp(broker.metadata.get("},{"line_number":1088,"context_line":" \u0027X-Container-Sysmeta-Policy-Change\u0027, [0])[0])"}],"source_content_type":"text/x-python","patch_set":9,"id":"17663a0b_96b09856","line":1085,"range":{"start_line":1085,"start_character":12,"end_line":1085,"end_character":47},"in_reply_to":"0114c1e8_80d6e0ab","updated":"2021-07-29 23:43:26.000000000","message":"Yeah, cool I see your point. And in fact if we\u0027re just tracking root put_timestamp there really doesn\u0027t need to be a metadata set on the root.. so that will clean things up a bit!\n\nWill rework it. That\u0027s for helping me not overthink it!","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ce97765f208019a9c380ffe86e4e9d707a2f7820","unresolved":true,"context_lines":[{"line_number":1098,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1099,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"},{"line_number":1100,"context_line":" broker.update_metadata("},{"line_number":1101,"context_line":" {\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":1102,"context_line":" (root_sp_timestamp.internal, Timestamp.now().internal)})"},{"line_number":1103,"context_line":""},{"line_number":1104,"context_line":" def _audit_shard_container(self, broker):"}],"source_content_type":"text/x-python","patch_set":9,"id":"8f2004be_f4fb71d0","line":1101,"range":{"start_line":1101,"start_character":18,"end_line":1101,"end_character":51},"updated":"2021-07-29 17:57:22.000000000","message":"I\u0027d prefer this to be named for whatever it is. So if it ends up being the root put timestamp, let\u0027s call it X-Container-Sysmeta-Root-Put-Timestamp\n\nI still need to reassure myself that put timestamp is the right one vs status-changed-at.","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":true,"context_lines":[{"line_number":1098,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1099,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"},{"line_number":1100,"context_line":" broker.update_metadata("},{"line_number":1101,"context_line":" {\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":1102,"context_line":" (root_sp_timestamp.internal, Timestamp.now().internal)})"},{"line_number":1103,"context_line":""},{"line_number":1104,"context_line":" def _audit_shard_container(self, broker):"}],"source_content_type":"text/x-python","patch_set":9,"id":"2617242e_ca311e2d","line":1101,"range":{"start_line":1101,"start_character":18,"end_line":1101,"end_character":51},"in_reply_to":"8f2004be_f4fb71d0","updated":"2021-07-29 23:43:26.000000000","message":"I\u0027ll change it to root-put-timestamp for now, at least until we confirm it is the data we want. But a delete and then an PUT should get a new put ts. And the migration tool also sets a new put ts, which is why I use it.","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29932970f21a07ac3eca27622d2d9730b733a64c","unresolved":true,"context_lines":[{"line_number":1121,"context_line":" # container; when not shrinking to root, but to another acceptor,"},{"line_number":1122,"context_line":" # the root range should be in sharded state and will not interfere"},{"line_number":1123,"context_line":" # with cleaving, listing or updating behaviour."},{"line_number":1124,"context_line":" shard_ranges, root_resp \u003d self._fetch_shard_ranges("},{"line_number":1125,"context_line":" broker, newest\u003dTrue,"},{"line_number":1126,"context_line":" params\u003d{\u0027marker\u0027: str_to_wsgi(own_shard_range.lower_str),"},{"line_number":1127,"context_line":" \u0027end_marker\u0027: str_to_wsgi(own_shard_range.upper_str),"}],"source_content_type":"text/x-python","patch_set":9,"id":"26a47168_54c227db","line":1124,"updated":"2021-07-29 21:57:24.000000000","message":"i don\u0027t like the effect the tuple return here has on the implementation of _fetch_shard_ranges at all; please consider refactoring this:\n\n resp \u003d _do_request_to_root()\n shard_ranges \u003d _parse_and_check_ranges(resp.body)\n new_awesome_stuff \u003d _do_brilliant_things(resp.headers)","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":true,"context_lines":[{"line_number":1121,"context_line":" # container; when not shrinking to root, but to another acceptor,"},{"line_number":1122,"context_line":" # the root range should be in sharded state and will not interfere"},{"line_number":1123,"context_line":" # with cleaving, listing or updating behaviour."},{"line_number":1124,"context_line":" shard_ranges, root_resp \u003d self._fetch_shard_ranges("},{"line_number":1125,"context_line":" broker, newest\u003dTrue,"},{"line_number":1126,"context_line":" params\u003d{\u0027marker\u0027: str_to_wsgi(own_shard_range.lower_str),"},{"line_number":1127,"context_line":" \u0027end_marker\u0027: str_to_wsgi(own_shard_range.upper_str),"}],"source_content_type":"text/x-python","patch_set":9,"id":"fa19f305_50abd57b","line":1124,"in_reply_to":"26a47168_54c227db","updated":"2021-07-29 23:43:26.000000000","message":"Thanks clay for pushing me to pick up my game and do it cleaner and stop being lazy (then just return the resp) your right!","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29932970f21a07ac3eca27622d2d9730b733a64c","unresolved":true,"context_lines":[{"line_number":1138,"context_line":" own_shard_range_from_root \u003d shard_range"},{"line_number":1139,"context_line":" # Only potentially update storage_policy_index if"},{"line_number":1140,"context_line":" # we find the SR with the root."},{"line_number":1141,"context_line":" self._update_storage_policy_index(broker, root_resp)"},{"line_number":1142,"context_line":" break"},{"line_number":1143,"context_line":" else:"},{"line_number":1144,"context_line":" # this is not necessarily an error - some replicas of the"}],"source_content_type":"text/x-python","patch_set":9,"id":"f41b1528_bbb6ff2d","line":1141,"updated":"2021-07-29 21:57:24.000000000","message":"can we get here when root_resp is None, the implemenation of _update_storage_policy_index does not seem to be robust to that","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":false,"context_lines":[{"line_number":1138,"context_line":" own_shard_range_from_root \u003d shard_range"},{"line_number":1139,"context_line":" # Only potentially update storage_policy_index if"},{"line_number":1140,"context_line":" # we find the SR with the root."},{"line_number":1141,"context_line":" self._update_storage_policy_index(broker, root_resp)"},{"line_number":1142,"context_line":" break"},{"line_number":1143,"context_line":" else:"},{"line_number":1144,"context_line":" # this is not necessarily an error - some replicas of the"}],"source_content_type":"text/x-python","patch_set":9,"id":"c97cb040_6bd985dc","line":1141,"in_reply_to":"f41b1528_bbb6ff2d","updated":"2021-07-29 23:43:26.000000000","message":"Ack","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29932970f21a07ac3eca27622d2d9730b733a64c","unresolved":true,"context_lines":[{"line_number":1436,"context_line":" # TODO: the root may not yet know about shard ranges to"},{"line_number":1437,"context_line":" # which a shard is sharding, but those could come from"},{"line_number":1438,"context_line":" # the broker"},{"line_number":1439,"context_line":" ranges, _ \u003d self._fetch_shard_ranges("},{"line_number":1440,"context_line":" broker, newest\u003dTrue,"},{"line_number":1441,"context_line":" params\u003d{\u0027states\u0027: \u0027updating\u0027,"},{"line_number":1442,"context_line":" \u0027marker\u0027: str_to_wsgi("}],"source_content_type":"text/x-python","patch_set":9,"id":"faa66cb2_0949d2e3","line":1439,"updated":"2021-07-29 21:57:24.000000000","message":"I could vomit.\n\nLet\u0027s maybe keep _fetch_shard_ranges() as a wrapper of:\n\n resp \u003d _do_request_to_root()\n shard_ranges \u003d _parse_and_check_ranges(resp.body)","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":true,"context_lines":[{"line_number":1436,"context_line":" # TODO: the root may not yet know about shard ranges to"},{"line_number":1437,"context_line":" # which a shard is sharding, but those could come from"},{"line_number":1438,"context_line":" # the broker"},{"line_number":1439,"context_line":" ranges, _ \u003d self._fetch_shard_ranges("},{"line_number":1440,"context_line":" broker, newest\u003dTrue,"},{"line_number":1441,"context_line":" params\u003d{\u0027states\u0027: \u0027updating\u0027,"},{"line_number":1442,"context_line":" \u0027marker\u0027: str_to_wsgi("}],"source_content_type":"text/x-python","patch_set":9,"id":"f5982857_3ddf50eb","line":1439,"in_reply_to":"faa66cb2_0949d2e3","updated":"2021-07-29 23:43:26.000000000","message":"Right again!","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":952,"context_line":""},{"line_number":953,"context_line":" resp \u003d self._fetch_shard_ranges_resp(broker, newest, params,"},{"line_number":954,"context_line":" include_deleted)"},{"line_number":955,"context_line":" return self._get_shard_ranges_from_response(broker, resp)"},{"line_number":956,"context_line":""},{"line_number":957,"context_line":" def _put_container(self, node, part, account, container, headers, body):"},{"line_number":958,"context_line":" try:"}],"source_content_type":"text/x-python","patch_set":11,"id":"44051cc6_b9603816","line":955,"updated":"2021-07-30 16:30:53.000000000","message":"love it!","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":1098,"context_line":" \u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":1099,"context_line":" shard_root_put_ts \u003d \\"},{"line_number":1100,"context_line":" Timestamp(broker.get_sharding_sysmeta(\u0027Root-Put-Timestamp\u0027) or 0)"},{"line_number":1101,"context_line":" if root_put_ts is None:"},{"line_number":1102,"context_line":" return"},{"line_number":1103,"context_line":" root_put_ts \u003d Timestamp(root_put_ts)"},{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."}],"source_content_type":"text/x-python","patch_set":11,"id":"af04248b_06e0e178","line":1102,"range":{"start_line":1101,"start_character":8,"end_line":1102,"end_character":18},"updated":"2021-07-30 16:30:53.000000000","message":"nit: this could go before line 1099\n\nor, we could include \u0027or root_sp_index is None\u0027 in the condition","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7f19c0a752f9ad4c5ec267f3f0ad270f7c339dbf","unresolved":false,"context_lines":[{"line_number":1098,"context_line":" \u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":1099,"context_line":" shard_root_put_ts \u003d \\"},{"line_number":1100,"context_line":" Timestamp(broker.get_sharding_sysmeta(\u0027Root-Put-Timestamp\u0027) or 0)"},{"line_number":1101,"context_line":" if root_put_ts is None:"},{"line_number":1102,"context_line":" return"},{"line_number":1103,"context_line":" root_put_ts \u003d Timestamp(root_put_ts)"},{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."}],"source_content_type":"text/x-python","patch_set":11,"id":"df1939a7_f5054ba7","line":1102,"range":{"start_line":1101,"start_character":8,"end_line":1102,"end_character":18},"in_reply_to":"af04248b_06e0e178","updated":"2021-08-02 00:54:56.000000000","message":"Done","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":1102,"context_line":" return"},{"line_number":1103,"context_line":" root_put_ts \u003d Timestamp(root_put_ts)"},{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."},{"line_number":1106,"context_line":" return"},{"line_number":1107,"context_line":" if root_sp_index is not None and \\"},{"line_number":1108,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"}],"source_content_type":"text/x-python","patch_set":11,"id":"b432d6e0_c77c4710","line":1105,"range":{"start_line":1105,"start_character":44,"end_line":1105,"end_character":47},"updated":"2021-07-30 16:30:53.000000000","message":"nit: s/as/is/","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7f19c0a752f9ad4c5ec267f3f0ad270f7c339dbf","unresolved":false,"context_lines":[{"line_number":1102,"context_line":" return"},{"line_number":1103,"context_line":" root_put_ts \u003d Timestamp(root_put_ts)"},{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."},{"line_number":1106,"context_line":" return"},{"line_number":1107,"context_line":" if root_sp_index is not None and \\"},{"line_number":1108,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"}],"source_content_type":"text/x-python","patch_set":11,"id":"0f96f223_37295687","line":1105,"range":{"start_line":1105,"start_character":44,"end_line":1105,"end_character":47},"in_reply_to":"b432d6e0_c77c4710","updated":"2021-08-02 00:54:56.000000000","message":"Done","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."},{"line_number":1106,"context_line":" return"},{"line_number":1107,"context_line":" if root_sp_index is not None and \\"},{"line_number":1108,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1109,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1110,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"}],"source_content_type":"text/x-python","patch_set":11,"id":"e6b5488a_c2f958ce","line":1107,"range":{"start_line":1107,"start_character":8,"end_line":1107,"end_character":37},"updated":"2021-07-30 16:30:53.000000000","message":"aha, so we check is not None here :) but could do this earlier","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7f19c0a752f9ad4c5ec267f3f0ad270f7c339dbf","unresolved":false,"context_lines":[{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp as newer then roots so ignore."},{"line_number":1106,"context_line":" return"},{"line_number":1107,"context_line":" if root_sp_index is not None and \\"},{"line_number":1108,"context_line":" root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1109,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1110,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"}],"source_content_type":"text/x-python","patch_set":11,"id":"92561a29_70035f34","line":1107,"range":{"start_line":1107,"start_character":8,"end_line":1107,"end_character":37},"in_reply_to":"e6b5488a_c2f958ce","updated":"2021-08-02 00:54:56.000000000","message":"Done","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":1600,"context_line":" else:"},{"line_number":1601,"context_line":" root_put_ts \u003d \\"},{"line_number":1602,"context_line":" broker.metadata.get("},{"line_number":1603,"context_line":" \u0027X-Container-Sysmeta-Shard-Root-Put-Timestamp\u0027)[0]"},{"line_number":1604,"context_line":" for shard_range in found_ranges:"},{"line_number":1605,"context_line":" self._increment_stat(\u0027created\u0027, \u0027attempted\u0027)"},{"line_number":1606,"context_line":" shard_range.update_state(ShardRange.CREATED)"}],"source_content_type":"text/x-python","patch_set":11,"id":"26a66300_d40ed7e5","line":1603,"range":{"start_line":1603,"start_character":67,"end_line":1603,"end_character":70},"updated":"2021-07-30 16:30:53.000000000","message":"-1, broker.metadata.get can return None, and is likely to immediately after upgrade","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"edad5f3826b26116a5751b77b4a13d25a73f13a6","unresolved":true,"context_lines":[{"line_number":1600,"context_line":" else:"},{"line_number":1601,"context_line":" root_put_ts \u003d \\"},{"line_number":1602,"context_line":" broker.metadata.get("},{"line_number":1603,"context_line":" \u0027X-Container-Sysmeta-Shard-Root-Put-Timestamp\u0027)[0]"},{"line_number":1604,"context_line":" for shard_range in found_ranges:"},{"line_number":1605,"context_line":" self._increment_stat(\u0027created\u0027, \u0027attempted\u0027)"},{"line_number":1606,"context_line":" shard_range.update_state(ShardRange.CREATED)"}],"source_content_type":"text/x-python","patch_set":11,"id":"96a93bfd_2ea8ca74","line":1603,"range":{"start_line":1603,"start_character":67,"end_line":1603,"end_character":70},"in_reply_to":"26a66300_d40ed7e5","updated":"2021-08-02 00:42:13.000000000","message":"yes, yes it could. I thought I covered for that.. but obviously I missed something. I think I used to have a default of something like [0] before.\n\nWill fix.","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"7f19c0a752f9ad4c5ec267f3f0ad270f7c339dbf","unresolved":true,"context_lines":[{"line_number":1600,"context_line":" else:"},{"line_number":1601,"context_line":" root_put_ts \u003d \\"},{"line_number":1602,"context_line":" broker.metadata.get("},{"line_number":1603,"context_line":" \u0027X-Container-Sysmeta-Shard-Root-Put-Timestamp\u0027)[0]"},{"line_number":1604,"context_line":" for shard_range in found_ranges:"},{"line_number":1605,"context_line":" self._increment_stat(\u0027created\u0027, \u0027attempted\u0027)"},{"line_number":1606,"context_line":" shard_range.update_state(ShardRange.CREATED)"}],"source_content_type":"text/x-python","patch_set":11,"id":"a77a4d6a_f0aa1db4","line":1603,"range":{"start_line":1603,"start_character":67,"end_line":1603,"end_character":70},"in_reply_to":"96a93bfd_2ea8ca74","updated":"2021-08-02 00:54:56.000000000","message":"Oh I see, yeah missed it here. and now that I\u0027ve moved it to shard sysmeta we can also now get away with something nicer like:\n\n root_put_ts \u003d \\\n broker.get_sharding_sysmeta(\u0027Root-Put-Timestamp\u0027) or 0","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ef2c4903c0ad5e37a2f4d54052001d376836495c","unresolved":true,"context_lines":[{"line_number":1610,"context_line":" broker.root_path),"},{"line_number":1611,"context_line":" \u0027X-Container-Sysmeta-Sharding\u0027: \u0027True\u0027,"},{"line_number":1612,"context_line":" \u0027X-Backend-Auto-Create\u0027: \u0027True\u0027,"},{"line_number":1613,"context_line":" \u0027X-Container-Sysmeta-Shard-Root-Put-Timestamp\u0027: root_put_ts}"},{"line_number":1614,"context_line":" # NB: we *used* to send along"},{"line_number":1615,"context_line":" # \u0027X-Container-Sysmeta-Shard-Root\u0027: broker.root_path"},{"line_number":1616,"context_line":" # but that isn\u0027t safe for container names with nulls or newlines"}],"source_content_type":"text/x-python","patch_set":11,"id":"bb34da9f_615964a8","line":1613,"updated":"2021-07-30 16:30:53.000000000","message":"yes!","commit_id":"10f1dd8e5f311a7e97f01917503eb951a2591f4e"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":1093,"context_line":" return True"},{"line_number":1094,"context_line":""},{"line_number":1095,"context_line":" def _update_storage_policy_index(self, broker, root_resp):"},{"line_number":1096,"context_line":" root_sp_index \u003d root_resp.headers.get(\u0027X-Backend-Storage-Policy-Index\u0027)"},{"line_number":1097,"context_line":" root_put_ts \u003d root_resp.headers.get("},{"line_number":1098,"context_line":" \u0027X-Backend-Put-Timestamp\u0027)"},{"line_number":1099,"context_line":" if root_put_ts is None or root_sp_index is None:"}],"source_content_type":"text/x-python","patch_set":18,"id":"b2d65562_8a646baf","line":1096,"range":{"start_line":1096,"start_character":8,"end_line":1096,"end_character":21},"updated":"2021-08-20 14:10:59.000000000","message":"this is a string...","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":1104,"context_line":" if shard_root_put_ts and shard_root_put_ts \u003e\u003d root_put_ts:"},{"line_number":1105,"context_line":" # local policy_change timestamp is newer then roots so ignore."},{"line_number":1106,"context_line":" return"},{"line_number":1107,"context_line":" if root_sp_index !\u003d broker.storage_policy_index:"},{"line_number":1108,"context_line":" self.logger.info(\"Migrating shard %s to storage_policy_index \""},{"line_number":1109,"context_line":" \"%d\", broker.db_file, int(root_sp_index))"},{"line_number":1110,"context_line":" broker.migrate_to_storage_policy(root_sp_index, Timestamp.now())"}],"source_content_type":"text/x-python","patch_set":18,"id":"3bca7335_573ddc30","line":1107,"range":{"start_line":1107,"start_character":11,"end_line":1107,"end_character":55},"updated":"2021-08-20 14:10:59.000000000","message":"hmmm, this is comparing a string with an int so it would always fail??? but the probe test passes :/\n\nwe should coerce root_sp_index to an int and validate that it is a valid policy","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"}],"test/probe/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4962a3cabeb2f5c57f16fc9d88e461b51c2e58ad","unresolved":true,"context_lines":[{"line_number":2790,"context_line":" self.assertEqual(root_broker.storage_policy_index,"},{"line_number":2791,"context_line":" new_policy_idx)"},{"line_number":2792,"context_line":""},{"line_number":2793,"context_line":" # Root brokers need to have a X-Container-Sysmeta-Policy-Change set"},{"line_number":2794,"context_line":" # or shard migration wont happen so let\u0027s add the metadata"},{"line_number":2795,"context_line":" now \u003d Timestamp.now().internal"},{"line_number":2796,"context_line":" root_broker.update_metadata({\u0027X-Container-Sysmeta-Policy-Change\u0027:"},{"line_number":2797,"context_line":" (now, now)})"}],"source_content_type":"text/x-python","patch_set":8,"id":"fb0425aa_9c87ad20","line":2794,"range":{"start_line":2793,"start_character":8,"end_line":2794,"end_character":66},"updated":"2021-07-28 14:24:28.000000000","message":"how does this happen in practice?","commit_id":"da376375df2d7cf6c1d3d7ead09cb7e945de5aa9"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"29932970f21a07ac3eca27622d2d9730b733a64c","unresolved":true,"context_lines":[{"line_number":2780,"context_line":" # now let\u0027s change the root_containers policy"},{"line_number":2781,"context_line":" new_policy_idx \u003d [pol.idx for pol in ENABLED_POLICIES"},{"line_number":2782,"context_line":" if pol !\u003d self.policy][0]"},{"line_number":2783,"context_line":" root_broker.migrate_to_storage_policy(new_policy_idx, Timestamp.now())"},{"line_number":2784,"context_line":""},{"line_number":2785,"context_line":" # replicate it out"},{"line_number":2786,"context_line":" self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":9,"id":"8b487974_be5c04d2","line":2783,"updated":"2021-07-29 21:57:24.000000000","message":"could one effect this change simply by adding the X-Container-Sysmeta-Policy-Change-Timestamp sysmeta via API?","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"4794ef821b64d85ba74809f90b46bbd2ec472941","unresolved":true,"context_lines":[{"line_number":2780,"context_line":" # now let\u0027s change the root_containers policy"},{"line_number":2781,"context_line":" new_policy_idx \u003d [pol.idx for pol in ENABLED_POLICIES"},{"line_number":2782,"context_line":" if pol !\u003d self.policy][0]"},{"line_number":2783,"context_line":" root_broker.migrate_to_storage_policy(new_policy_idx, Timestamp.now())"},{"line_number":2784,"context_line":""},{"line_number":2785,"context_line":" # replicate it out"},{"line_number":2786,"context_line":" self.replicators.once()"}],"source_content_type":"text/x-python","patch_set":9,"id":"c476d05d_263eefb1","line":2783,"in_reply_to":"8b487974_be5c04d2","updated":"2021-07-29 23:43:26.000000000","message":"Well I need the root to have a different policy index, so just using the helper method to change it. When the sharder runs below the shard will look at the root put_ts and the change will happen.\n\nOr maybe I am missing what your saying.. it is pre-coffee :P","commit_id":"12fa27551b76fa5bacaf9e4211ae5958a7e3420e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"838ab97145ad48678e6f6f07b80024b0d9b31afe","unresolved":true,"context_lines":[{"line_number":2245,"context_line":" self.put_objects(all_obj_names)"},{"line_number":2246,"context_line":""},{"line_number":2247,"context_line":" # XXX: this is unexpected (to me)"},{"line_number":2248,"context_line":" self.assert_container_listing([])"},{"line_number":2249,"context_line":" self.assert_container_object_count(0)"},{"line_number":2250,"context_line":" self.sharders.once()"},{"line_number":2251,"context_line":" self.assert_container_listing(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":12,"id":"2bc28c2b_34fb0bb2","line":2248,"updated":"2021-07-30 20:58:56.000000000","message":"I think what\u0027s going on here is the objects ARE in the shards - but with the \"wrong\" spi from the shard container\u0027s perspective - I think there is also a risk of the reconilcer running here and mis-queueing the rows.","commit_id":"ed8c78d95ed425a2d47c93f229fa93b2520fce8a"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"edad5f3826b26116a5751b77b4a13d25a73f13a6","unresolved":true,"context_lines":[{"line_number":2245,"context_line":" self.put_objects(all_obj_names)"},{"line_number":2246,"context_line":""},{"line_number":2247,"context_line":" # XXX: this is unexpected (to me)"},{"line_number":2248,"context_line":" self.assert_container_listing([])"},{"line_number":2249,"context_line":" self.assert_container_object_count(0)"},{"line_number":2250,"context_line":" self.sharders.once()"},{"line_number":2251,"context_line":" self.assert_container_listing(all_obj_names)"}],"source_content_type":"text/x-python","patch_set":12,"id":"0734762f_6fb356f9","line":2248,"in_reply_to":"2bc28c2b_34fb0bb2","updated":"2021-08-02 00:42:13.000000000","message":"hmm, yeah, that\u0027s interesting behaviour! Not sure what the best thing is, I need to think about it.\nMaybe the reconciler needs to be more shard aware. Or rather the replicator that enqueues the queue. There will always be edge cases where the shard DB may fall behind the root \n\nMaybe 1. shard dbs should look at the coming spi in a request (I\u0027ll double check we\u0027re sending it) and return obj results based on it (or maybe better, ensure we return all objects that match query even if they aren\u0027t apart of the main spi).\n2. enqueuing shard objs onto the reconciler queue actually does a root lookup first to find the expected spi (or look at the latest x-container-sysmeta-shard-root-put-timestamp pick the spi of the container (sender or reciever)that holds that metadata).","commit_id":"ed8c78d95ed425a2d47c93f229fa93b2520fce8a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"838ab97145ad48678e6f6f07b80024b0d9b31afe","unresolved":true,"context_lines":[{"line_number":2248,"context_line":" self.assert_container_listing([])"},{"line_number":2249,"context_line":" self.assert_container_object_count(0)"},{"line_number":2250,"context_line":" self.sharders.once()"},{"line_number":2251,"context_line":" self.assert_container_listing(all_obj_names)"},{"line_number":2252,"context_line":" self.assert_container_object_count(len(all_obj_names))"},{"line_number":2253,"context_line":""},{"line_number":2254,"context_line":" # and there reconciler work"}],"source_content_type":"text/x-python","patch_set":12,"id":"3271c776_df220eb3","line":2251,"updated":"2021-07-30 20:58:56.000000000","message":"on master this fails - the shards don\u0027t update their spi and so they keep shadowing the ec object rows.","commit_id":"ed8c78d95ed425a2d47c93f229fa93b2520fce8a"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"838ab97145ad48678e6f6f07b80024b0d9b31afe","unresolved":true,"context_lines":[{"line_number":2259,"context_line":" for container in int_client.iter_containers(MISPLACED_OBJECTS_ACCOUNT):"},{"line_number":2260,"context_line":" for obj in int_client.iter_objects(MISPLACED_OBJECTS_ACCOUNT,"},{"line_number":2261,"context_line":" container[\u0027name\u0027]):"},{"line_number":2262,"context_line":" self.assertEqual(obj[\u0027content_type\u0027], \u0027application/x-delete\u0027)"},{"line_number":2263,"context_line":" prefix \u003d \u0027%s:/%s/%s/\u0027 % ("},{"line_number":2264,"context_line":" int(self.policy), self.account, self.container_name)"},{"line_number":2265,"context_line":" self.assertTrue(obj[\u0027name\u0027].startswith(prefix))"}],"source_content_type":"text/x-python","patch_set":12,"id":"4f28baa9_8943f872","line":2262,"updated":"2021-07-30 20:58:56.000000000","message":"all the reconciler entries are for migrating the tombstones","commit_id":"ed8c78d95ed425a2d47c93f229fa93b2520fce8a"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"f54f59d44de64036b3cd72944451fc71d9cc3b04","unresolved":true,"context_lines":[{"line_number":2254,"context_line":" self.assert_container_listing(all_obj_names)"},{"line_number":2255,"context_line":" self.assert_container_object_count(len(all_obj_names))"},{"line_number":2256,"context_line":""},{"line_number":2257,"context_line":" # and there reconciler work"},{"line_number":2258,"context_line":" found \u003d []"},{"line_number":2259,"context_line":" self.replicators.once()"},{"line_number":2260,"context_line":" Manager([\u0027container-updater\u0027]).once()"}],"source_content_type":"text/x-python","patch_set":15,"id":"7deda855_1d1b654f","line":2257,"updated":"2021-08-13 14:26:33.000000000","message":"ok so this looks surprising at first but the work is actually for the old policy tombstones, not for the more recently put objects","commit_id":"04febd6590d80a3c389cd8fa4f0d8f85bed9fd30"}],"test/unit/container/test_sharder.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"bb4e6b2061f51a5cf0ccfc61a3b43d17dfeb09d8","unresolved":true,"context_lines":[{"line_number":5516,"context_line":" info_lines \u003d sharder.logger.get_lines_for_level(\u0027info\u0027)"},{"line_number":5517,"context_line":" self.assertIn(log_msg, info_lines)"},{"line_number":5518,"context_line":" self.assertEqual(broker.storage_policy_index, 1)"},{"line_number":5519,"context_line":" expected_put_ts \u003d Timestamp(root_put_ts, offset\u003d2)"},{"line_number":5520,"context_line":" expected_delete_ts \u003d Timestamp(root_put_ts, offset\u003d1)"},{"line_number":5521,"context_line":" info \u003d broker.get_info()"},{"line_number":5522,"context_line":" self.assertEqual(expected_put_ts.internal, info[\u0027put_timestamp\u0027])"},{"line_number":5523,"context_line":" self.assertEqual(expected_put_ts.internal, info[\u0027status_changed_at\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"c9600fb8_73042af7","line":5520,"range":{"start_line":5519,"start_character":8,"end_line":5520,"end_character":61},"updated":"2021-07-22 11:08:55.000000000","message":"so the shard broker takes on the put and delete timestamps of the root?","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"e8186b811739a67eb688773cd8b451b15c34057f","unresolved":true,"context_lines":[{"line_number":5516,"context_line":" info_lines \u003d sharder.logger.get_lines_for_level(\u0027info\u0027)"},{"line_number":5517,"context_line":" self.assertIn(log_msg, info_lines)"},{"line_number":5518,"context_line":" self.assertEqual(broker.storage_policy_index, 1)"},{"line_number":5519,"context_line":" expected_put_ts \u003d Timestamp(root_put_ts, offset\u003d2)"},{"line_number":5520,"context_line":" expected_delete_ts \u003d Timestamp(root_put_ts, offset\u003d1)"},{"line_number":5521,"context_line":" info \u003d broker.get_info()"},{"line_number":5522,"context_line":" self.assertEqual(expected_put_ts.internal, info[\u0027put_timestamp\u0027])"},{"line_number":5523,"context_line":" self.assertEqual(expected_put_ts.internal, info[\u0027status_changed_at\u0027])"}],"source_content_type":"text/x-python","patch_set":3,"id":"f55303a3_eec11d41","line":5520,"range":{"start_line":5519,"start_character":8,"end_line":5520,"end_character":61},"in_reply_to":"c9600fb8_73042af7","updated":"2021-07-23 03:07:21.000000000","message":"Yeah, In my mind I thought we\u0027d want the root and shard to be considered the same... but if this causes issues with timestamps we can revisit.","commit_id":"2d91ef276229ce88c7cdc2d6bfc6e5cdbe8d4bae"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":5580,"context_line":" self.assertGreater(info[\u0027delete_timestamp\u0027], orig_put_ts)"},{"line_number":5581,"context_line":" else:"},{"line_number":5582,"context_line":" # nothing changed"},{"line_number":5583,"context_line":" self.assertNotIn(log_msg, info_lines)"},{"line_number":5584,"context_line":" self.assertEqual(broker.storage_policy_index, 0)"},{"line_number":5585,"context_line":""},{"line_number":5586,"context_line":" # first the wont changes."}],"source_content_type":"text/x-python","patch_set":18,"id":"ae719fd8_4487c9b2","line":5583,"updated":"2021-08-20 14:10:59.000000000","message":"but if we fix the headers to have str values then the logs do actually contain:\n\n\u0027Migrating shard %s to storage_policy_index 0\u0027\n\nbecause \u00270\u0027 !\u003d 0","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"d624c3974b1dd8ca8893b4a610d32af8b1a122e1","unresolved":true,"context_lines":[{"line_number":5586,"context_line":" # first the wont changes."},{"line_number":5587,"context_line":" # It wont change if there isn\u0027t a put_timestamp from the root"},{"line_number":5588,"context_line":" extra_response_headers \u003d {"},{"line_number":5589,"context_line":" \u0027X-Backend-Storage-Policy-Index\u0027: 1}"},{"line_number":5590,"context_line":" extra_shard_meta \u003d {"},{"line_number":5591,"context_line":" \u0027X-Container-Sysmeta-Shard-Root-Put-Timestamp\u0027:"},{"line_number":5592,"context_line":" (next(self.ts_iter).internal, Timestamp.now().internal)}"}],"source_content_type":"text/x-python","patch_set":18,"id":"c1784607_9c7e1827","line":5589,"range":{"start_line":5589,"start_character":46,"end_line":5589,"end_character":47},"updated":"2021-08-20 14:10:59.000000000","message":"header values are always strings","commit_id":"f41b02dd953dc5ef10be177cb3d5ced961784e65"}]}