)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"e854b176447b28dd2a98847b6fd391294719c926","unresolved":false,"context_lines":[{"line_number":19,"context_line":""},{"line_number":20,"context_line":"By default, with this patch, the object server will set"},{"line_number":21,"context_line":"require_durable\u003dTrue when opening disk files for GETs and,"},{"line_number":22,"context_line":"resulting in no change to existing behavior."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Change-Id: I2310981fd1c4622ff5d1a739cbcc59637ffe3fc3"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"da20952f_87d237ff","line":22,"updated":"2015-08-28 04:22:33.000000000","message":"I thought we were going to do the exact opposite?  Be optimistic right!?","commit_id":"0813d47ec2996b17d0be938a0f69ec6681e7e899"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab62daa1f93e16a6d770c89aa4801a6fb791be9e","unresolved":false,"context_lines":[{"line_number":19,"context_line":""},{"line_number":20,"context_line":"By default, with this patch, the object server will set"},{"line_number":21,"context_line":"require_durable\u003dTrue when opening disk files for GETs and,"},{"line_number":22,"context_line":"resulting in no change to existing behavior."},{"line_number":23,"context_line":""},{"line_number":24,"context_line":"Change-Id: I2310981fd1c4622ff5d1a739cbcc59637ffe3fc3"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":4,"id":"da20952f_74898ac9","line":22,"in_reply_to":"da20952f_87d237ff","updated":"2015-09-11 17:09:43.000000000","message":"Yes. I\u0027m just making it clear here that this patch won\u0027t break anything while the proxy has no support.\n\nPeluse and I had a plan to split the work between object server and proxy changes, which is why this patch just has half the job done. Whoever does get to the proxy part first can build on this hopefully.","commit_id":"0813d47ec2996b17d0be938a0f69ec6681e7e899"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"5f397239912c7a96d25e5da5758b6fe33ccc822e","unresolved":false,"context_lines":[{"line_number":12,"context_line":"  a durable file for the fragment archive that they return"},{"line_number":13,"context_line":"  in response to a GET. The proxy will now be satisfied if"},{"line_number":14,"context_line":"  just one object server has a durable file at the same"},{"line_number":15,"context_line":"  timestamp as fragments from other object servers."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"  This means that the proxy can now successfully GET an"},{"line_number":18,"context_line":"  an object that had missing durable files when it was PUT."}],"source_content_type":"text/x-gerrit-commit-message","patch_set":12,"id":"9a1a9d01_ba82fbfe","line":15,"updated":"2015-09-27 13:54:07.000000000","message":"We may even reduce this to no durable file being required to respond 2xx on GET - just a quorum of frags at same timestamp.","commit_id":"bb5b406a2b00272e182d48745f7aea2b057aa398"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":15,"context_line":"  timestamp as fragments from other object servers."},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"  This means that the proxy can now successfully GET an"},{"line_number":18,"context_line":"  an object that had missing durable files when it was PUT."},{"line_number":19,"context_line":""},{"line_number":20,"context_line":"- The proxy will now ensure that its has a quorum of *unique*"},{"line_number":21,"context_line":"  fragment indexes from object servers before considering a"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":16,"id":"9a1a9d01_92e9b7e8","line":18,"updated":"2015-09-30 12:56:04.000000000","message":"(Nits) redundant \"an\"","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"}],"doc/source/overview_erasure_code.rst":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":503,"context_line":"#. The proxy server makes simultaneous requests to participating nodes."},{"line_number":504,"context_line":"#. As soon as the proxy has `ec_ndata` distinct fragment archives at the same"},{"line_number":505,"context_line":"   timestamp, and an indication from at least one object server that a"},{"line_number":506,"context_line":"   `ts.durable` file exists for that timestamp, it calls on PyECLib to decode"},{"line_number":507,"context_line":"   the data."},{"line_number":508,"context_line":"#. The proxy streams the decoded data it has back to the client."},{"line_number":509,"context_line":"#. Repeat until the proxy is done sending data back to the client."}],"source_content_type":"text/x-rst","patch_set":29,"id":"5a5ae5dd_ab33fff4","line":506,"range":{"start_line":506,"start_character":3,"end_line":506,"end_character":14},"updated":"2016-02-08 10:29:54.000000000","message":"Nits) timestamp.durable, strictly. (to clarify \"it\u0027s not tombstone\")","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":503,"context_line":"#. The proxy server makes simultaneous requests to participating nodes."},{"line_number":504,"context_line":"#. As soon as the proxy has `ec_ndata` distinct fragment archives at the same"},{"line_number":505,"context_line":"   timestamp, and an indication from at least one object server that a"},{"line_number":506,"context_line":"   `ts.durable` file exists for that timestamp, it calls on PyECLib to decode"},{"line_number":507,"context_line":"   the data."},{"line_number":508,"context_line":"#. The proxy streams the decoded data it has back to the client."},{"line_number":509,"context_line":"#. Repeat until the proxy is done sending data back to the client."}],"source_content_type":"text/x-rst","patch_set":29,"id":"3a57f1b5_911bedc9","line":506,"range":{"start_line":506,"start_character":3,"end_line":506,"end_character":14},"in_reply_to":"5a5ae5dd_ab33fff4","updated":"2016-02-12 18:21:33.000000000","message":"Done","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":518,"context_line":"file for the fragment archive that they return in response to a GET. The proxy"},{"line_number":519,"context_line":"will be satisfied if just one object server has a `.durable` file at the same"},{"line_number":520,"context_line":"timestamp as fragment archives returned from other object servers. This means"},{"line_number":521,"context_line":"that the proxy can successfully GET an an object that had missing `.durable`"},{"line_number":522,"context_line":"files when it was PUT (i.e. a partial PUT failure occurred)."},{"line_number":523,"context_line":""},{"line_number":524,"context_line":"The proxy may receive fragment archives having different timestamps, and may"}],"source_content_type":"text/x-rst","patch_set":29,"id":"5a5ae5dd_2ba4ef05","line":521,"range":{"start_line":521,"start_character":36,"end_line":521,"end_character":41},"updated":"2016-02-08 10:29:54.000000000","message":"Nits) s/an an/an/","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":518,"context_line":"file for the fragment archive that they return in response to a GET. The proxy"},{"line_number":519,"context_line":"will be satisfied if just one object server has a `.durable` file at the same"},{"line_number":520,"context_line":"timestamp as fragment archives returned from other object servers. This means"},{"line_number":521,"context_line":"that the proxy can successfully GET an an object that had missing `.durable`"},{"line_number":522,"context_line":"files when it was PUT (i.e. a partial PUT failure occurred)."},{"line_number":523,"context_line":""},{"line_number":524,"context_line":"The proxy may receive fragment archives having different timestamps, and may"}],"source_content_type":"text/x-rst","patch_set":29,"id":"3a57f1b5_11861ddb","line":521,"range":{"start_line":521,"start_character":36,"end_line":521,"end_character":41},"in_reply_to":"5a5ae5dd_2ba4ef05","updated":"2016-02-12 18:21:33.000000000","message":"Done","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":521,"context_line":"receive several fragment archives having the same index. The proxy therefore"},{"line_number":522,"context_line":"ensures that its has sufficient fragment archives with the same timestamp"},{"line_number":523,"context_line":"and distinct fragment indexes before considering a GET to be successful."},{"line_number":524,"context_line":""},{"line_number":525,"context_line":"Object Server"},{"line_number":526,"context_line":"-------------"},{"line_number":527,"context_line":""}],"source_content_type":"text/x-rst","patch_set":39,"id":"fa7ab95a_63e23f03","line":524,"updated":"2016-09-07 12:04:21.000000000","message":"memo) is it necessary to add also about different indexes may be in an object-server?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":521,"context_line":"receive several fragment archives having the same index. The proxy therefore"},{"line_number":522,"context_line":"ensures that its has sufficient fragment archives with the same timestamp"},{"line_number":523,"context_line":"and distinct fragment indexes before considering a GET to be successful."},{"line_number":524,"context_line":""},{"line_number":525,"context_line":"Object Server"},{"line_number":526,"context_line":"-------------"},{"line_number":527,"context_line":""}],"source_content_type":"text/x-rst","patch_set":39,"id":"9a89bdaa_bfc44192","line":524,"in_reply_to":"fa7ab95a_63e23f03","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":518,"context_line":"when it was PUT (i.e. a partial PUT failure occurred)."},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"Note also that an object server may inform the proxy server that it has more"},{"line_number":521,"context_line":"than one EC archive for different timestamps and/or fragment indexes, which may"},{"line_number":522,"context_line":"cause the proxy server to issue multiple requests for distinct EC archives to"},{"line_number":523,"context_line":"that object server. (This situation can temporarily occur after a ring"},{"line_number":524,"context_line":"rebalance when a handoff node storing an archive has become a primary node and"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_a8f9d44f","line":521,"range":{"start_line":521,"start_character":24,"end_line":521,"end_character":44},"updated":"2016-09-14 03:15:04.000000000","message":"How does the proxy decide which timestamp to use? The latest one? The latest one that actually has a .durable and enough available fragments?\n\nHow does this interact with .meta? There, do we take latest (non-sys) meta, always?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":518,"context_line":"when it was PUT (i.e. a partial PUT failure occurred)."},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"Note also that an object server may inform the proxy server that it has more"},{"line_number":521,"context_line":"than one EC archive for different timestamps and/or fragment indexes, which may"},{"line_number":522,"context_line":"cause the proxy server to issue multiple requests for distinct EC archives to"},{"line_number":523,"context_line":"that object server. (This situation can temporarily occur after a ring"},{"line_number":524,"context_line":"rebalance when a handoff node storing an archive has become a primary node and"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_81c11fe7","line":521,"range":{"start_line":521,"start_character":24,"end_line":521,"end_character":44},"in_reply_to":"7a8ec9b2_564d3871","updated":"2016-09-14 17:32:23.000000000","message":"As Kota said: proxy uses latest timestamp for which there is a complete set of ec_ndata fragments, including at least one that is durable. The timestamp is the \"data timestamp\" when that is available from upgraded object servers.\n\nWe have discussed (and agreed) that we could drop the requirement for even a single durable, but this patch hasn\u0027t taken that step.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":518,"context_line":"when it was PUT (i.e. a partial PUT failure occurred)."},{"line_number":519,"context_line":""},{"line_number":520,"context_line":"Note also that an object server may inform the proxy server that it has more"},{"line_number":521,"context_line":"than one EC archive for different timestamps and/or fragment indexes, which may"},{"line_number":522,"context_line":"cause the proxy server to issue multiple requests for distinct EC archives to"},{"line_number":523,"context_line":"that object server. (This situation can temporarily occur after a ring"},{"line_number":524,"context_line":"rebalance when a handoff node storing an archive has become a primary node and"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_564d3871","line":521,"range":{"start_line":521,"start_character":24,"end_line":521,"end_character":44},"in_reply_to":"7a8ec9b2_a8f9d44f","updated":"2016-09-14 10:37:45.000000000","message":"How to decide:\nthe newest of responses bucket (response bucket is a collection of responses which has same timestamp) which has enough fragments to decode.\n\nMetadata:\nRight now, tha last one of collected fragment archive response[2]. A TODO thing which select the newest metadata of them are remaing but I think it can be in another improvement. \n\n1: https://review.openstack.org/#/c/215276/40/swift/proxy/controllers/obj.py@1990\n2: https://review.openstack.org/#/c/215276/40/swift/proxy/controllers/obj.py@1862","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":522,"context_line":"cause the proxy server to issue multiple requests for distinct EC archives to"},{"line_number":523,"context_line":"that object server. (This situation can temporarily occur after a ring"},{"line_number":524,"context_line":"rebalance when a handoff node storing an archive has become a primary node and"},{"line_number":525,"context_line":"received it\u0027s primary archive but not yet moved the handoff archive to it\u0027s"},{"line_number":526,"context_line":"primary node.)"},{"line_number":527,"context_line":""},{"line_number":528,"context_line":"The proxy may receive EC archives having different timestamps, and may"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_686a6c1c","line":525,"updated":"2016-09-14 03:15:04.000000000","message":"nit: \"its\" not \"it\u0027s\" in both places.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":522,"context_line":"cause the proxy server to issue multiple requests for distinct EC archives to"},{"line_number":523,"context_line":"that object server. (This situation can temporarily occur after a ring"},{"line_number":524,"context_line":"rebalance when a handoff node storing an archive has become a primary node and"},{"line_number":525,"context_line":"received it\u0027s primary archive but not yet moved the handoff archive to it\u0027s"},{"line_number":526,"context_line":"primary node.)"},{"line_number":527,"context_line":""},{"line_number":528,"context_line":"The proxy may receive EC archives having different timestamps, and may"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_814f5f91","line":525,"in_reply_to":"7a8ec9b2_686a6c1c","updated":"2016-09-14 17:32:23.000000000","message":"Done","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":527,"context_line":""},{"line_number":528,"context_line":"The proxy may receive EC archives having different timestamps, and may"},{"line_number":529,"context_line":"receive several EC archives having the same index. The proxy therefore"},{"line_number":530,"context_line":"ensures that its has sufficient EC archives with the same timestamp"},{"line_number":531,"context_line":"and distinct fragment indexes before considering a GET to be successful."},{"line_number":532,"context_line":""},{"line_number":533,"context_line":"Object Server"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_21304b01","line":530,"updated":"2016-09-14 17:32:23.000000000","message":"s/its/it/","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":528,"context_line":"The proxy may receive EC archives having different timestamps, and may"},{"line_number":529,"context_line":"receive several EC archives having the same index. The proxy therefore"},{"line_number":530,"context_line":"ensures that its has sufficient EC archives with the same timestamp"},{"line_number":531,"context_line":"and distinct fragment indexes before considering a GET to be successful."},{"line_number":532,"context_line":""},{"line_number":533,"context_line":"Object Server"},{"line_number":534,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_889338fa","line":531,"range":{"start_line":531,"start_character":61,"end_line":531,"end_character":71},"updated":"2016-09-14 03:15:04.000000000","message":"Somewhat out of scope, but what does failure look like? 404? 5xx?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":528,"context_line":"The proxy may receive EC archives having different timestamps, and may"},{"line_number":529,"context_line":"receive several EC archives having the same index. The proxy therefore"},{"line_number":530,"context_line":"ensures that its has sufficient EC archives with the same timestamp"},{"line_number":531,"context_line":"and distinct fragment indexes before considering a GET to be successful."},{"line_number":532,"context_line":""},{"line_number":533,"context_line":"Object Server"},{"line_number":534,"context_line":"-------------"}],"source_content_type":"text/x-rst","patch_set":40,"id":"7a8ec9b2_fc744cd0","line":531,"range":{"start_line":531,"start_character":61,"end_line":531,"end_character":71},"in_reply_to":"7a8ec9b2_889338fa","updated":"2016-09-14 17:32:23.000000000","message":"the client response status depends on the mix of object server status codes - could be a 503, 404, 416...","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"swift/common/utils.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"358410d16db7bd464481ef3bdadd2fe88ff9e5a1","unresolved":false,"context_lines":[{"line_number":828,"context_line":"        return cmp(self.internal, other.internal)"},{"line_number":829,"context_line":""},{"line_number":830,"context_line":"    def __hash__(self):"},{"line_number":831,"context_line":"        return self.internal.__hash__()"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":""},{"line_number":834,"context_line":"def normalize_timestamp(timestamp):"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_cd05e914","line":831,"updated":"2015-10-02 06:09:39.000000000","message":"I think hash(self.internal)","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c2044b092e281e255e762578edd75d20fcbd60f","unresolved":false,"context_lines":[{"line_number":828,"context_line":"        return cmp(self.internal, other.internal)"},{"line_number":829,"context_line":""},{"line_number":830,"context_line":"    def __hash__(self):"},{"line_number":831,"context_line":"        return self.internal.__hash__()"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":""},{"line_number":834,"context_line":"def normalize_timestamp(timestamp):"}],"source_content_type":"text/x-python","patch_set":20,"id":"3a29b11f_755bfe54","line":831,"in_reply_to":"9a1a9d01_cd05e914","updated":"2015-10-20 14:23:39.000000000","message":"Done","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03c7df4898b047111aebb4592f6422acc8f19d13","unresolved":false,"context_lines":[{"line_number":828,"context_line":"        return cmp(self.internal, other.internal)"},{"line_number":829,"context_line":""},{"line_number":830,"context_line":"    def __hash__(self):"},{"line_number":831,"context_line":"        return self.internal.__hash__()"},{"line_number":832,"context_line":""},{"line_number":833,"context_line":""},{"line_number":834,"context_line":"def normalize_timestamp(timestamp):"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_05d4feba","line":831,"in_reply_to":"9a1a9d01_cd05e914","updated":"2015-10-02 17:37:22.000000000","message":"what, is hash, like, a builtin? :D:D","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"358410d16db7bd464481ef3bdadd2fe88ff9e5a1","unresolved":false,"context_lines":[{"line_number":3753,"context_line":"            return json.loads(value)"},{"line_number":3754,"context_line":"        except (TypeError, ValueError):"},{"line_number":3755,"context_line":"            pass"},{"line_number":3756,"context_line":"    return default"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_88feff03","line":3756,"updated":"2015-10-02 06:09:39.000000000","message":"... interesting ...","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9f423ee02141db8c71e4c4a0ae6e86761919c3d9","unresolved":false,"context_lines":[{"line_number":4097,"context_line":"            return json.loads(value)"},{"line_number":4098,"context_line":"        except (TypeError, ValueError):"},{"line_number":4099,"context_line":"            pass"},{"line_number":4100,"context_line":"    return None"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_f85d237a","line":4100,"updated":"2016-09-15 00:04:00.000000000","message":"I didn\u0027t see any tests - but this looks obviously correct and best i can tell it is\n\n1. https://gist.github.com/clayg/de52b3c970eb1b33c1eb00755b07d416","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"swift/obj/diskfile.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f832c2009aa370f89e75db03e1e32d099d9511d","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        If a file is considered part of a valid on-disk file set it will be"},{"line_number":512,"context_line":"        added to the results dict, keyed by its extension. The newest file that"},{"line_number":513,"context_line":"        confirms a valid fileset as being durable is added to the results under"},{"line_number":514,"context_line":"        the key \u0027newest_durable\u0027. If a file is considered to be obsolete it"},{"line_number":515,"context_line":"        will be added to a list stored under the key \u0027obsolete\u0027 in the results."},{"line_number":516,"context_line":""},{"line_number":517,"context_line":"        :param files: a list of file names."}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_b110cea9","line":514,"updated":"2015-09-15 19:18:49.000000000","message":"newest_durable may not be the cleverest of names because its not necessarily a durable file. suggestions?","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1ed91bacb04552b64455cf813f0d4515383b0d24","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        If a file is considered part of a valid on-disk file set it will be"},{"line_number":512,"context_line":"        added to the results dict, keyed by its extension. The newest file that"},{"line_number":513,"context_line":"        confirms a valid fileset as being durable is added to the results under"},{"line_number":514,"context_line":"        the key \u0027newest_durable\u0027. If a file is considered to be obsolete it"},{"line_number":515,"context_line":"        will be added to a list stored under the key \u0027obsolete\u0027 in the results."},{"line_number":516,"context_line":""},{"line_number":517,"context_line":"        :param files: a list of file names."}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_1d7ac5bd","line":514,"in_reply_to":"ba15a1d1_88f798f4","updated":"2015-09-16 19:11:17.000000000","message":"changed name to oldest_valid","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ec1575a884c0a49ffb9b27f17a4dff1e5794ba44","unresolved":false,"context_lines":[{"line_number":511,"context_line":"        If a file is considered part of a valid on-disk file set it will be"},{"line_number":512,"context_line":"        added to the results dict, keyed by its extension. The newest file that"},{"line_number":513,"context_line":"        confirms a valid fileset as being durable is added to the results under"},{"line_number":514,"context_line":"        the key \u0027newest_durable\u0027. If a file is considered to be obsolete it"},{"line_number":515,"context_line":"        will be added to a list stored under the key \u0027obsolete\u0027 in the results."},{"line_number":516,"context_line":""},{"line_number":517,"context_line":"        :param files: a list of file names."}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_88f798f4","line":514,"in_reply_to":"ba15a1d1_b110cea9","updated":"2015-09-15 22:45:13.000000000","message":"being a common base class implementation - it feels leaky for this docstring to have to discuss .durable files?","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ec1575a884c0a49ffb9b27f17a4dff1e5794ba44","unresolved":false,"context_lines":[{"line_number":558,"context_line":"                pass"},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"            if results.get(\u0027found_valid\u0027) and not include_obsolete:"},{"line_number":561,"context_line":"                break"},{"line_number":562,"context_line":""},{"line_number":563,"context_line":"        if verify:"},{"line_number":564,"context_line":"            assert self._verify_on_disk_files("}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_ad614e3b","line":561,"updated":"2015-09-15 22:45:13.000000000","message":"I don\u0027t really know what\u0027s going on here - this method used to be a nice simple iterator over that farmed out the file handling to the implementations in _gather_on_disk_file.\n\nThe common code between DiskFileManager._gather_on_disk_file and ECDiskFileManager._gather_on_disk_file should be extracted out of those methods into a common helper rather than coupled into the iterator.","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1ed91bacb04552b64455cf813f0d4515383b0d24","unresolved":false,"context_lines":[{"line_number":558,"context_line":"                pass"},{"line_number":559,"context_line":""},{"line_number":560,"context_line":"            if results.get(\u0027found_valid\u0027) and not include_obsolete:"},{"line_number":561,"context_line":"                break"},{"line_number":562,"context_line":""},{"line_number":563,"context_line":"        if verify:"},{"line_number":564,"context_line":"            assert self._verify_on_disk_files("}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_e04c08ca","line":561,"in_reply_to":"ba15a1d1_ad614e3b","updated":"2015-09-16 19:11:17.000000000","message":"OK\n\nIt ended up like this because first there was scope to share some of the code between the two subclasses, and then lumping it into this method saves some bouncing up and down between subclass method and superclass helper.\n\n(well, closer to truth is I didn\u0027t quite see what you\u0027re suggesting, I was just glad to remove redundancy!)\n\nwill change as you suggest","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":544,"context_line":"                pass"},{"line_number":545,"context_line":"        for ext in exts:"},{"line_number":546,"context_line":"            exts[ext] \u003d sorted("},{"line_number":547,"context_line":"                exts[ext], key\u003dlambda info: info[\u0027timestamp\u0027], reverse\u003dTrue)"},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"        # the results  dict is used to collect results of file filtering"},{"line_number":550,"context_line":"        results \u003d {}"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_2347861e","line":547,"updated":"2015-09-22 22:00:39.000000000","message":"you might could do a sorted insert - bisect module I think?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":544,"context_line":"                pass"},{"line_number":545,"context_line":"        for ext in exts:"},{"line_number":546,"context_line":"            exts[ext] \u003d sorted("},{"line_number":547,"context_line":"                exts[ext], key\u003dlambda info: info[\u0027timestamp\u0027], reverse\u003dTrue)"},{"line_number":548,"context_line":""},{"line_number":549,"context_line":"        # the results  dict is used to collect results of file filtering"},{"line_number":550,"context_line":"        results \u003d {}"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_64afa52a","line":547,"in_reply_to":"ba15a1d1_2347861e","updated":"2015-09-23 20:03:04.000000000","message":"From what I read, bisect insort_right|left is useful to avoid sorting a list after each insertion but does involve a list.insert() for every insort_left. I\u0027m only sorting each list once, which I think ends up more efficient than doing multiple inserts.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":572,"context_line":"        # transform discard info to a list of files"},{"line_number":573,"context_line":"        for file_info in results.get(\u0027discard\u0027, []):"},{"line_number":574,"context_line":"            results.setdefault(\u0027obsolete\u0027, []).append(file_info[\u0027filename\u0027])"},{"line_number":575,"context_line":"        results.pop(\u0027discard\u0027, None)"},{"line_number":576,"context_line":""},{"line_number":577,"context_line":"        if verify:"},{"line_number":578,"context_line":"            assert self._verify_on_disk_files("}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_c3f1eace","line":575,"updated":"2015-09-22 22:00:39.000000000","message":"meh, better to just update the code that\u0027s dealing with discard to accept the richer data structure","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":572,"context_line":"        # transform discard info to a list of files"},{"line_number":573,"context_line":"        for file_info in results.get(\u0027discard\u0027, []):"},{"line_number":574,"context_line":"            results.setdefault(\u0027obsolete\u0027, []).append(file_info[\u0027filename\u0027])"},{"line_number":575,"context_line":"        results.pop(\u0027discard\u0027, None)"},{"line_number":576,"context_line":""},{"line_number":577,"context_line":"        if verify:"},{"line_number":578,"context_line":"            assert self._verify_on_disk_files("}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_c4647123","line":575,"in_reply_to":"ba15a1d1_c3f1eace","updated":"2015-09-23 20:03:04.000000000","message":"Good idea. Done.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1990,"context_line":"        for ext in exts.keys():"},{"line_number":1991,"context_line":"            exts[ext], older \u003d self._split_by_time("},{"line_number":1992,"context_line":"                exts[ext], exts[\u0027.data\u0027][:1], keep_same\u003dTrue)"},{"line_number":1993,"context_line":"            results.setdefault(\u0027discard\u0027, []).extend(older)"},{"line_number":1994,"context_line":""},{"line_number":1995,"context_line":"        # set results"},{"line_number":1996,"context_line":"        if exts[\u0027.data\u0027]:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_037da21b","line":1993,"updated":"2015-09-22 22:00:39.000000000","message":"I don\u0027t understand the difference between discard, obsolete, and is_reclaimable.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":1990,"context_line":"        for ext in exts.keys():"},{"line_number":1991,"context_line":"            exts[ext], older \u003d self._split_by_time("},{"line_number":1992,"context_line":"                exts[ext], exts[\u0027.data\u0027][:1], keep_same\u003dTrue)"},{"line_number":1993,"context_line":"            results.setdefault(\u0027discard\u0027, []).extend(older)"},{"line_number":1994,"context_line":""},{"line_number":1995,"context_line":"        # set results"},{"line_number":1996,"context_line":"        if exts[\u0027.data\u0027]:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_dae3e8d7","line":1993,"in_reply_to":"ba15a1d1_037da21b","updated":"2015-09-23 20:03:04.000000000","message":"\u0027discard\u0027 \u003d\u003d \u0027obsolete\u0027 and will go away in next version (thanks!).\n\nobsolete files are deleted NQA. is_reclaimable is used when a delete is only made after reclaim age.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2119,"context_line":"        if frag_index is not None:"},{"line_number":2120,"context_line":"            self._frag_index \u003d self.manager.validate_fragment_index(frag_index)"},{"line_number":2121,"context_line":"        self.require_durable \u003d (kwargs[\u0027require_durable\u0027]"},{"line_number":2122,"context_line":"                                if \u0027require_durable\u0027 in kwargs else True)"},{"line_number":2123,"context_line":""},{"line_number":2124,"context_line":"    def _get_ondisk_file(self, files):"},{"line_number":2125,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_183bcc39","line":2122,"updated":"2015-09-22 22:00:39.000000000","message":"I don\u0027t see how this is different than:\n\n    kwargs.get(\u0027require_durable\u0027, True)","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2119,"context_line":"        if frag_index is not None:"},{"line_number":2120,"context_line":"            self._frag_index \u003d self.manager.validate_fragment_index(frag_index)"},{"line_number":2121,"context_line":"        self.require_durable \u003d (kwargs[\u0027require_durable\u0027]"},{"line_number":2122,"context_line":"                                if \u0027require_durable\u0027 in kwargs else True)"},{"line_number":2123,"context_line":""},{"line_number":2124,"context_line":"    def _get_ondisk_file(self, files):"},{"line_number":2125,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_5ada78e9","line":2122,"in_reply_to":"ba15a1d1_183bcc39","updated":"2015-09-23 20:03:04.000000000","message":"Done","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2134,"context_line":"        if not results.get(\u0027is_durable\u0027):"},{"line_number":2135,"context_line":"            self._durable \u003d False"},{"line_number":2136,"context_line":"        if \u0027durable_info\u0027 in results:"},{"line_number":2137,"context_line":"            self._durable_timestamp \u003d results[\u0027durable_info\u0027][\u0027timestamp\u0027]"},{"line_number":2138,"context_line":"        return results"},{"line_number":2139,"context_line":""},{"line_number":2140,"context_line":"    def purge(self, timestamp, frag_index):"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_7860c835","line":2137,"updated":"2015-09-22 22:00:39.000000000","message":"do we need both `._durable` and `._durable_timestamp`?\n\nCan you have `._durable` True and `._durable_timestamp` still None?\n\nCan we get rid of one of the two pieces of state and reduce the chance of a conflict/bug?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2134,"context_line":"        if not results.get(\u0027is_durable\u0027):"},{"line_number":2135,"context_line":"            self._durable \u003d False"},{"line_number":2136,"context_line":"        if \u0027durable_info\u0027 in results:"},{"line_number":2137,"context_line":"            self._durable_timestamp \u003d results[\u0027durable_info\u0027][\u0027timestamp\u0027]"},{"line_number":2138,"context_line":"        return results"},{"line_number":2139,"context_line":""},{"line_number":2140,"context_line":"    def purge(self, timestamp, frag_index):"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_2aef2fb2","line":2137,"in_reply_to":"ba15a1d1_7860c835","updated":"2015-09-23 20:03:04.000000000","message":"Yes we can get rid of self._durable. I will also change this to try to be more obvious what is happening w.r.t the frag sets being used.\n\nSomehow I need the proxy to be able to distinguish between an obj server that has not been upgraded to this patch vs an object server that has no durable file. So I needed more than just returning X-Backend-Durable-Timestamp.\n\nIn next version I introduce X-Backend-Data-Timestamp (because the X-Backend-Timestamp may have the time of a .meta file :/) and the absence of that header will allow proxy to figure out whether we have no durable or are just old code.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2260,"context_line":"                                if an associated durable file is found. If"},{"line_number":2261,"context_line":"                                False, a valid fileset may be gathered using"},{"line_number":2262,"context_line":"                                the newest data file even if it has no"},{"line_number":2263,"context_line":"                                associated durable file."},{"line_number":2264,"context_line":"        \"\"\""},{"line_number":2265,"context_line":"        # discard everything older than most recent .durable"},{"line_number":2266,"context_line":"        for ext in exts.keys():"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_d8f79457","line":2263,"updated":"2015-09-22 22:00:39.000000000","message":"when do we use `require_durable\u003dTrue` - in most situations I can imagine a proxy making a subsequent GET for a different frag archive stream it\u0027s going to have a specific ask based on the available frag/timestamp/durable\u0027s that were reported as available when it decided to reject the previous stream.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2260,"context_line":"                                if an associated durable file is found. If"},{"line_number":2261,"context_line":"                                False, a valid fileset may be gathered using"},{"line_number":2262,"context_line":"                                the newest data file even if it has no"},{"line_number":2263,"context_line":"                                associated durable file."},{"line_number":2264,"context_line":"        \"\"\""},{"line_number":2265,"context_line":"        # discard everything older than most recent .durable"},{"line_number":2266,"context_line":"        for ext in exts.keys():"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_2aa72f51","line":2263,"in_reply_to":"ba15a1d1_d8f79457","updated":"2015-09-23 20:03:04.000000000","message":"Agree, requiring durable is less relevant than requiring a specific *timestamp*, once proxy knows what\u0027s on backends*. But I haven\u0027t quite made it to the second proxy targetted GET specifying \u0027I want this frag index at this *timestamp*, nothing else!\u0027.\n\n#AddToWishList\n\n*Of course we might need to consider the backends changing state between the first optimistic GET and a subsequent targetted GET :(","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2278,"context_line":"        while all_frags:"},{"line_number":2279,"context_line":"            frag_set, all_frags \u003d self._split_by_time("},{"line_number":2280,"context_line":"                all_frags, all_frags[:1], keep_same\u003dTrue)"},{"line_number":2281,"context_line":"            all_frag_sets.append(frag_set)"},{"line_number":2282,"context_line":"            if (durable_info"},{"line_number":2283,"context_line":"                    and durable_info[\u0027timestamp\u0027] \u003d\u003d frag_set[0][\u0027timestamp\u0027]):"},{"line_number":2284,"context_line":"                durable_frag_set \u003d frag_set"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ab760c3c","line":2281,"updated":"2015-09-22 22:00:39.000000000","message":"gah so dense, what does _split_by_time do?  what is the type structure of all_frags_sets (list of sets?)  what is the cardinality (one for each... timestamp?)  Maybe it would be clearer if all_frag_sets was a dict with a descriptive key instead just the index...","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2278,"context_line":"        while all_frags:"},{"line_number":2279,"context_line":"            frag_set, all_frags \u003d self._split_by_time("},{"line_number":2280,"context_line":"                all_frags, all_frags[:1], keep_same\u003dTrue)"},{"line_number":2281,"context_line":"            all_frag_sets.append(frag_set)"},{"line_number":2282,"context_line":"            if (durable_info"},{"line_number":2283,"context_line":"                    and durable_info[\u0027timestamp\u0027] \u003d\u003d frag_set[0][\u0027timestamp\u0027]):"},{"line_number":2284,"context_line":"                durable_frag_set \u003d frag_set"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_13ef021f","line":2281,"in_reply_to":"ba15a1d1_ab760c3c","updated":"2015-09-23 20:03:04.000000000","message":"Done: added docstring to split_by_time, made all_frag_sets a dict keyed by timestamp","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2287,"context_line":"        if require_durable:"},{"line_number":2288,"context_line":"            frag_set \u003d durable_frag_set"},{"line_number":2289,"context_line":"        elif all_frag_sets:"},{"line_number":2290,"context_line":"            frag_set \u003d all_frag_sets[0]"},{"line_number":2291,"context_line":"        else:"},{"line_number":2292,"context_line":"            frag_set \u003d None"},{"line_number":2293,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ab492cd2","line":2290,"updated":"2015-09-22 22:00:39.000000000","message":"oic, it\u0027s ordered by time - but all_frags is already ordered...","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2301,"context_line":"            else:"},{"line_number":2302,"context_line":"                frag \u003d sorted("},{"line_number":2303,"context_line":"                    frag_set, key\u003dlambda info: info[\u0027frag_index\u0027],"},{"line_number":2304,"context_line":"                    reverse\u003dTrue)"},{"line_number":2305,"context_line":""},{"line_number":2306,"context_line":"        # set results"},{"line_number":2307,"context_line":"        # TODO: pass the all_frag_sets data structure back to disk file class"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_abe5aca8","line":2304,"updated":"2015-09-22 22:00:39.000000000","message":"OMG I\u0027ve totally lost it :\u0027(\n\n... can not grok anything more in this method :\u0027(","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2301,"context_line":"            else:"},{"line_number":2302,"context_line":"                frag \u003d sorted("},{"line_number":2303,"context_line":"                    frag_set, key\u003dlambda info: info[\u0027frag_index\u0027],"},{"line_number":2304,"context_line":"                    reverse\u003dTrue)"},{"line_number":2305,"context_line":""},{"line_number":2306,"context_line":"        # set results"},{"line_number":2307,"context_line":"        # TODO: pass the all_frag_sets data structure back to disk file class"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_d3f42af2","line":2304,"in_reply_to":"ba15a1d1_abe5aca8","updated":"2015-09-23 20:03:04.000000000","message":"added some comments","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2337,"context_line":"        Verify that the final combination of on disk files complies with the"},{"line_number":2338,"context_line":"        diskfile contract."},{"line_number":2339,"context_line":""},{"line_number":2340,"context_line":"        :param results: files that have been found and accepted"},{"line_number":2341,"context_line":"        :param frag_index: specifies a specific fragment index .data file"},{"line_number":2342,"context_line":"        :returns: True if the file combination is compliant, False otherwise"},{"line_number":2343,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ab4e0cbf","line":2340,"updated":"2015-09-22 22:00:39.000000000","message":"is this such a better name than accepted_files that it\u0027s worth having the named arguments different from the replicated diskfile?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2337,"context_line":"        Verify that the final combination of on disk files complies with the"},{"line_number":2338,"context_line":"        diskfile contract."},{"line_number":2339,"context_line":""},{"line_number":2340,"context_line":"        :param results: files that have been found and accepted"},{"line_number":2341,"context_line":"        :param frag_index: specifies a specific fragment index .data file"},{"line_number":2342,"context_line":"        :returns: True if the file combination is compliant, False otherwise"},{"line_number":2343,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_8a5c3b4c","line":2340,"in_reply_to":"ba15a1d1_ab4e0cbf","updated":"2015-09-23 20:03:04.000000000","message":"probably not :) Will fix","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        into two: items newer than any in other_file_info and items older than"},{"line_number":503,"context_line":"        any in other_file_info."},{"line_number":504,"context_line":"        :param file_info_list: a list if file_info dicts"},{"line_number":505,"context_line":"        :param other_file_info_list: a list if file_info dicts"},{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_12bcc7da","line":505,"updated":"2015-09-30 12:56:04.000000000","message":"(Nits) a list OF file_info dicts?","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":502,"context_line":"        into two: items newer than any in other_file_info and items older than"},{"line_number":503,"context_line":"        any in other_file_info."},{"line_number":504,"context_line":"        :param file_info_list: a list if file_info dicts"},{"line_number":505,"context_line":"        :param other_file_info_list: a list if file_info dicts"},{"line_number":506,"context_line":"        :param split_right: if True then items with the same timestamp as the"},{"line_number":507,"context_line":"                            split time will be included in the first returned"},{"line_number":508,"context_line":"                            list. otherwise those items will be included in the"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_6a7c15ec","line":505,"in_reply_to":"9a1a9d01_12bcc7da","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":512,"context_line":"        \"\"\""},{"line_number":513,"context_line":"        done \u003d (1, 0)"},{"line_number":514,"context_line":"        if split_right:"},{"line_number":515,"context_line":"            done \u003d (1,)"},{"line_number":516,"context_line":"        for i, item in enumerate(file_info_list):"},{"line_number":517,"context_line":"            for file_info in other_file_info_list:"},{"line_number":518,"context_line":"                if cmp(file_info[\u0027timestamp\u0027], item[\u0027timestamp\u0027]) in done:"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_32c1cb54","line":515,"updated":"2015-09-30 12:56:04.000000000","message":"(memo) (1, 0) means equal to or greater than, and (1, ) means greater than.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":512,"context_line":"        \"\"\""},{"line_number":513,"context_line":"        done \u003d (1, 0)"},{"line_number":514,"context_line":"        if split_right:"},{"line_number":515,"context_line":"            done \u003d (1,)"},{"line_number":516,"context_line":"        for i, item in enumerate(file_info_list):"},{"line_number":517,"context_line":"            for file_info in other_file_info_list:"},{"line_number":518,"context_line":"                if cmp(file_info[\u0027timestamp\u0027], item[\u0027timestamp\u0027]) in done:"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_ea2905e7","line":515,"in_reply_to":"9a1a9d01_32c1cb54","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":537,"context_line":"                  filename extension; a list of obsolete files stored under the"},{"line_number":538,"context_line":"                  key \u0027obsolete\u0027."},{"line_number":539,"context_line":"        \"\"\""},{"line_number":540,"context_line":"        files.sort(reverse\u003dTrue)"},{"line_number":541,"context_line":"        # Build the exts data structure:"},{"line_number":542,"context_line":"        # exts is a dict that maps file extensions to a list of file_info"},{"line_number":543,"context_line":"        # dicts for the files having that extension. The file_info dicts are of"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_d2b16f02","line":540,"updated":"2015-09-30 12:56:04.000000000","message":"It  seems no need to sort anymore because exts dict will be sorted by timestamp later.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":537,"context_line":"                  filename extension; a list of obsolete files stored under the"},{"line_number":538,"context_line":"                  key \u0027obsolete\u0027."},{"line_number":539,"context_line":"        \"\"\""},{"line_number":540,"context_line":"        files.sort(reverse\u003dTrue)"},{"line_number":541,"context_line":"        # Build the exts data structure:"},{"line_number":542,"context_line":"        # exts is a dict that maps file extensions to a list of file_info"},{"line_number":543,"context_line":"        # dicts for the files having that extension. The file_info dicts are of"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_0a3eb927","line":540,"in_reply_to":"9a1a9d01_d2b16f02","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":561,"context_line":"            exts[ext] \u003d sorted("},{"line_number":562,"context_line":"                exts[ext], key\u003dlambda info: info[\u0027timestamp\u0027], reverse\u003dTrue)"},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"        # the results  dict is used to collect results of file filtering"},{"line_number":565,"context_line":"        results \u003d {}"},{"line_number":566,"context_line":""},{"line_number":567,"context_line":"        # non-tombstones older than or equal to latest tombstone are obsolete"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_f2b6f3f7","line":564,"updated":"2015-09-30 12:56:04.000000000","message":"(Nits) remove a duplicated space between \"results\" and \"dict\"","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":561,"context_line":"            exts[ext] \u003d sorted("},{"line_number":562,"context_line":"                exts[ext], key\u003dlambda info: info[\u0027timestamp\u0027], reverse\u003dTrue)"},{"line_number":563,"context_line":""},{"line_number":564,"context_line":"        # the results  dict is used to collect results of file filtering"},{"line_number":565,"context_line":"        results \u003d {}"},{"line_number":566,"context_line":""},{"line_number":567,"context_line":"        # non-tombstones older than or equal to latest tombstone are obsolete"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_2a593d8e","line":564,"in_reply_to":"9a1a9d01_f2b6f3f7","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2343,"context_line":"        .data file that is chosen to construct the disk file"},{"line_number":2344,"context_line":""},{"line_number":2345,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":2346,"context_line":"        :param results: a dict to hold results"},{"line_number":2347,"context_line":"        :param frag_index: if set, search for a specific fragment index .data"},{"line_number":2348,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2349,"context_line":"        :param frag_prefs: if set, search for any fragment index .data file"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_72c3c357","line":2346,"updated":"2015-09-30 12:56:04.000000000","message":"I\u0027d like to add the format of results. (i.e. what\u0027s included in the dict, they seems \".data\", \"frag_sets\", \"chosen_frag_sets\" and \"durable_frag_sets\"), it\u0027s ok on later patch tho.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2343,"context_line":"        .data file that is chosen to construct the disk file"},{"line_number":2344,"context_line":""},{"line_number":2345,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":2346,"context_line":"        :param results: a dict to hold results"},{"line_number":2347,"context_line":"        :param frag_index: if set, search for a specific fragment index .data"},{"line_number":2348,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2349,"context_line":"        :param frag_prefs: if set, search for any fragment index .data file"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_b671340e","line":2346,"in_reply_to":"9a1a9d01_72c3c357","updated":"2015-09-30 23:33:45.000000000","message":"There\u0027s some description in BaseDiskFileManager.gather_on_disk_files()","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":2343,"context_line":"        .data file that is chosen to construct the disk file"},{"line_number":2344,"context_line":""},{"line_number":2345,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":2346,"context_line":"        :param results: a dict to hold results"},{"line_number":2347,"context_line":"        :param frag_index: if set, search for a specific fragment index .data"},{"line_number":2348,"context_line":"                           file, otherwise accept the first valid .data file."},{"line_number":2349,"context_line":"        :param frag_prefs: if set, search for any fragment index .data file"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_44d2d801","line":2346,"in_reply_to":"9a1a9d01_b671340e","updated":"2015-10-05 07:08:59.000000000","message":"ok, that makes me sense.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2385,"context_line":"            # that timestamp? If so choose the highest of those frag_indexes."},{"line_number":2386,"context_line":"            for ts, ignore in [(pref[0], pref[1:])"},{"line_number":2387,"context_line":"                               for pref in frag_prefs"},{"line_number":2388,"context_line":"                               if pref[0] in candidate_frag_sets]:"},{"line_number":2389,"context_line":"                    available \u003d [info[\u0027frag_index\u0027]"},{"line_number":2390,"context_line":"                                 for info in candidate_frag_sets[ts]]"},{"line_number":2391,"context_line":"                    acceptable \u003d list(set(available).difference(set(ignore)))"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_32a8eb8c","line":2388,"updated":"2015-09-30 12:56:04.000000000","message":"It\u0027s okay but I like 4 spaces for indent rather than 8 spaces.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2385,"context_line":"            # that timestamp? If so choose the highest of those frag_indexes."},{"line_number":2386,"context_line":"            for ts, ignore in [(pref[0], pref[1:])"},{"line_number":2387,"context_line":"                               for pref in frag_prefs"},{"line_number":2388,"context_line":"                               if pref[0] in candidate_frag_sets]:"},{"line_number":2389,"context_line":"                    available \u003d [info[\u0027frag_index\u0027]"},{"line_number":2390,"context_line":"                                 for info in candidate_frag_sets[ts]]"},{"line_number":2391,"context_line":"                    acceptable \u003d list(set(available).difference(set(ignore)))"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_75887671","line":2388,"in_reply_to":"9a1a9d01_32a8eb8c","updated":"2015-09-30 23:33:45.000000000","message":"me too, thanks!","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2387,"context_line":"                               for pref in frag_prefs"},{"line_number":2388,"context_line":"                               if pref[0] in candidate_frag_sets]:"},{"line_number":2389,"context_line":"                    available \u003d [info[\u0027frag_index\u0027]"},{"line_number":2390,"context_line":"                                 for info in candidate_frag_sets[ts]]"},{"line_number":2391,"context_line":"                    acceptable \u003d list(set(available).difference(set(ignore)))"},{"line_number":2392,"context_line":"                    if acceptable:"},{"line_number":2393,"context_line":"                        chosen_frag_set \u003d candidate_frag_sets[ts]"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_d2a88f8e","line":2390,"updated":"2015-09-30 12:56:04.000000000","message":"(memo) availabe_indices strictly.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2387,"context_line":"                               for pref in frag_prefs"},{"line_number":2388,"context_line":"                               if pref[0] in candidate_frag_sets]:"},{"line_number":2389,"context_line":"                    available \u003d [info[\u0027frag_index\u0027]"},{"line_number":2390,"context_line":"                                 for info in candidate_frag_sets[ts]]"},{"line_number":2391,"context_line":"                    acceptable \u003d list(set(available).difference(set(ignore)))"},{"line_number":2392,"context_line":"                    if acceptable:"},{"line_number":2393,"context_line":"                        chosen_frag_set \u003d candidate_frag_sets[ts]"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_55a452b8","line":2390,"in_reply_to":"9a1a9d01_d2a88f8e","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2392,"context_line":"                    if acceptable:"},{"line_number":2393,"context_line":"                        chosen_frag_set \u003d candidate_frag_sets[ts]"},{"line_number":2394,"context_line":"                        # override any frag_index passed in as method param"},{"line_number":2395,"context_line":"                        frag_index \u003d acceptable[-1]"},{"line_number":2396,"context_line":"                        break"},{"line_number":2397,"context_line":"                    else:"},{"line_number":2398,"context_line":"                        # this frag_set has no acceptable frag index so"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_f2ad939b","line":2395,"updated":"2015-09-30 12:56:04.000000000","message":"(memo) frag_index is set as the last index of acceptable indices.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2392,"context_line":"                    if acceptable:"},{"line_number":2393,"context_line":"                        chosen_frag_set \u003d candidate_frag_sets[ts]"},{"line_number":2394,"context_line":"                        # override any frag_index passed in as method param"},{"line_number":2395,"context_line":"                        frag_index \u003d acceptable[-1]"},{"line_number":2396,"context_line":"                        break"},{"line_number":2397,"context_line":"                    else:"},{"line_number":2398,"context_line":"                        # this frag_set has no acceptable frag index so"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_d598627a","line":2395,"in_reply_to":"9a1a9d01_f2ad939b","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2408,"context_line":"        else:"},{"line_number":2409,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2410,"context_line":""},{"line_number":2411,"context_line":"        # Select a single chosen frag form the chosen frag_set, by either"},{"line_number":2412,"context_line":"        # matching against a specified frag_index or taking the highest index."},{"line_number":2413,"context_line":"        chosen_frag \u003d None"},{"line_number":2414,"context_line":"        if chosen_frag_set:"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_b294fb42","line":2411,"updated":"2015-09-30 12:56:04.000000000","message":"(Nits) s/form/from/","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2408,"context_line":"        else:"},{"line_number":2409,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2410,"context_line":""},{"line_number":2411,"context_line":"        # Select a single chosen frag form the chosen frag_set, by either"},{"line_number":2412,"context_line":"        # matching against a specified frag_index or taking the highest index."},{"line_number":2413,"context_line":"        chosen_frag \u003d None"},{"line_number":2414,"context_line":"        if chosen_frag_set:"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_5592124d","line":2411,"in_reply_to":"9a1a9d01_b294fb42","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"358410d16db7bd464481ef3bdadd2fe88ff9e5a1","unresolved":false,"context_lines":[{"line_number":2377,"context_line":"        preferences: the first item in these per-timestamp lists list should be"},{"line_number":2378,"context_line":"        a string representation of a timestamp and subsequent items in that"},{"line_number":2379,"context_line":"        list should be valid fragment indexes (i.e. whole numbers) that should"},{"line_number":2380,"context_line":"        be EXCLUDED when choosing a fragment at that timestamp."},{"line_number":2381,"context_line":""},{"line_number":2382,"context_line":"        The order of per-timestamp sub-lists in the frag_prefs list is"},{"line_number":2383,"context_line":"        significant and indicates descending preference for fragments from each"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_a8518310","line":2380,"updated":"2015-10-02 06:09:39.000000000","message":"a list of lists, and the type of the first element in the sublist isn\u0027t the same as the type of the rest :\\\n\nI guess when I heard json I was thinking we\u0027d get a rich self-descriptive dataformat\n\n[\n    {\n        \u0027timestamp\u0027: \u0027\u003ctime\u003e\u0027,\n        \u0027frag_prefs\u0027: [1, 3, 9]\n    }\n]\n\n... maybe it\u0027s a trade off of space on the wire?  like but we\u0027re talking *bytes* right?  I think something k/v based has a much better shot on longevity...","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c2044b092e281e255e762578edd75d20fcbd60f","unresolved":false,"context_lines":[{"line_number":2377,"context_line":"        preferences: the first item in these per-timestamp lists list should be"},{"line_number":2378,"context_line":"        a string representation of a timestamp and subsequent items in that"},{"line_number":2379,"context_line":"        list should be valid fragment indexes (i.e. whole numbers) that should"},{"line_number":2380,"context_line":"        be EXCLUDED when choosing a fragment at that timestamp."},{"line_number":2381,"context_line":""},{"line_number":2382,"context_line":"        The order of per-timestamp sub-lists in the frag_prefs list is"},{"line_number":2383,"context_line":"        significant and indicates descending preference for fragments from each"}],"source_content_type":"text/x-python","patch_set":20,"id":"3a29b11f_95a01a45","line":2380,"in_reply_to":"9a1a9d01_a8518310","updated":"2015-10-20 14:23:39.000000000","message":"Done","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03c7df4898b047111aebb4592f6422acc8f19d13","unresolved":false,"context_lines":[{"line_number":2377,"context_line":"        preferences: the first item in these per-timestamp lists list should be"},{"line_number":2378,"context_line":"        a string representation of a timestamp and subsequent items in that"},{"line_number":2379,"context_line":"        list should be valid fragment indexes (i.e. whole numbers) that should"},{"line_number":2380,"context_line":"        be EXCLUDED when choosing a fragment at that timestamp."},{"line_number":2381,"context_line":""},{"line_number":2382,"context_line":"        The order of per-timestamp sub-lists in the frag_prefs list is"},{"line_number":2383,"context_line":"        significant and indicates descending preference for fragments from each"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_489e8bb9","line":2380,"in_reply_to":"9a1a9d01_a8518310","updated":"2015-10-02 17:37:22.000000000","message":"fair point. I was fretting too much about the size of the header value. I agree that a dict for each timestamp is much more intuitive. But the top level list is important to preserve preference order.","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"358410d16db7bd464481ef3bdadd2fe88ff9e5a1","unresolved":false,"context_lines":[{"line_number":2390,"context_line":"        construct the diskfile."},{"line_number":2391,"context_line":""},{"line_number":2392,"context_line":"        When a frag_prefs arg is provided, including and empty list, there is"},{"line_number":2393,"context_line":"        no requirement for there to durable file at the same timestamp as a"},{"line_number":2394,"context_line":"        .data file that is chosen to construct the disk file"},{"line_number":2395,"context_line":""},{"line_number":2396,"context_line":"        :param exts: dict of lists of file info, keyed by extension"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_c84ab755","line":2393,"updated":"2015-10-02 06:09:39.000000000","message":"s/for there to/for there to be a/","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c2044b092e281e255e762578edd75d20fcbd60f","unresolved":false,"context_lines":[{"line_number":2390,"context_line":"        construct the diskfile."},{"line_number":2391,"context_line":""},{"line_number":2392,"context_line":"        When a frag_prefs arg is provided, including and empty list, there is"},{"line_number":2393,"context_line":"        no requirement for there to durable file at the same timestamp as a"},{"line_number":2394,"context_line":"        .data file that is chosen to construct the disk file"},{"line_number":2395,"context_line":""},{"line_number":2396,"context_line":"        :param exts: dict of lists of file info, keyed by extension"}],"source_content_type":"text/x-python","patch_set":20,"id":"3a29b11f_b5a3d64e","line":2393,"in_reply_to":"9a1a9d01_c84ab755","updated":"2015-10-20 14:23:39.000000000","message":"Done","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2212,"context_line":"        self._frag_prefs \u003d self._validate_frag_prefs(kwargs.get(\u0027frag_prefs\u0027))"},{"line_number":2213,"context_line":"        self._durable_frag_set \u003d None"},{"line_number":2214,"context_line":""},{"line_number":2215,"context_line":"    def _validate_frag_prefs(self, frag_prefs):"},{"line_number":2216,"context_line":"        # We *do* want to preserve an empty frag_prefs list because it"},{"line_number":2217,"context_line":"        # indicates that a durable file is not required."},{"line_number":2218,"context_line":"        if frag_prefs is None:"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_4ee939b6","line":2215,"updated":"2016-02-08 10:29:54.000000000","message":"memo) This method also does the translation from timestamp str(?) to Timestamp instance in addition to the validation. Nothing else modified in the response.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":2212,"context_line":"        self._frag_prefs \u003d self._validate_frag_prefs(kwargs.get(\u0027frag_prefs\u0027))"},{"line_number":2213,"context_line":"        self._durable_frag_set \u003d None"},{"line_number":2214,"context_line":""},{"line_number":2215,"context_line":"    def _validate_frag_prefs(self, frag_prefs):"},{"line_number":2216,"context_line":"        # We *do* want to preserve an empty frag_prefs list because it"},{"line_number":2217,"context_line":"        # indicates that a durable file is not required."},{"line_number":2218,"context_line":"        if frag_prefs is None:"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_1485ab05","line":2215,"in_reply_to":"5a5ae5dd_4ee939b6","updated":"2016-02-12 18:21:33.000000000","message":"Done","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2237,"context_line":"                result.append({\u0027timestamp\u0027: timestamp, \u0027exclude\u0027: exclusions})"},{"line_number":2238,"context_line":"        except (KeyError, TypeError) as e:"},{"line_number":2239,"context_line":"                    raise DiskFileError("},{"line_number":2240,"context_line":"                        \u0027Bad frag_prefs: %r: %s\u0027 % (frag_prefs, e))"},{"line_number":2241,"context_line":"        return result"},{"line_number":2242,"context_line":""},{"line_number":2243,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_8ed97159","line":2240,"updated":"2016-02-08 10:29:54.000000000","message":"nits) 4 white spaces are better for indent.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":2237,"context_line":"                result.append({\u0027timestamp\u0027: timestamp, \u0027exclude\u0027: exclusions})"},{"line_number":2238,"context_line":"        except (KeyError, TypeError) as e:"},{"line_number":2239,"context_line":"                    raise DiskFileError("},{"line_number":2240,"context_line":"                        \u0027Bad frag_prefs: %r: %s\u0027 % (frag_prefs, e))"},{"line_number":2241,"context_line":"        return result"},{"line_number":2242,"context_line":""},{"line_number":2243,"context_line":"    @property"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_d47b9326","line":2240,"in_reply_to":"5a5ae5dd_8ed97159","updated":"2016-02-12 18:21:33.000000000","message":"Done","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2479,"context_line":"            for ts, ignore_indexes in ["},{"line_number":2480,"context_line":"                    (ts_pref[\u0027timestamp\u0027], ts_pref[\u0027exclude\u0027])"},{"line_number":2481,"context_line":"                    for ts_pref in frag_prefs"},{"line_number":2482,"context_line":"                    if ts_pref[\u0027timestamp\u0027] in candidate_frag_sets]:"},{"line_number":2483,"context_line":"                available_indexes \u003d [info[\u0027frag_index\u0027]"},{"line_number":2484,"context_line":"                                     for info in candidate_frag_sets[ts]]"},{"line_number":2485,"context_line":"                acceptable_indexes \u003d list(set(available_indexes)"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_bf4fb932","line":2482,"updated":"2016-02-08 10:29:54.000000000","message":"memo) I thought, it might be unsafe to compare with Timestamp instance and Timestamp str but it\u0027s safe so no matter if one dict has Timestamp instance keys and the other has Timestamp str keys.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d11dc9a684ab8e8658e0e7c4fd6fc71c36f483e","unresolved":false,"context_lines":[{"line_number":2479,"context_line":"            for ts, ignore_indexes in ["},{"line_number":2480,"context_line":"                    (ts_pref[\u0027timestamp\u0027], ts_pref[\u0027exclude\u0027])"},{"line_number":2481,"context_line":"                    for ts_pref in frag_prefs"},{"line_number":2482,"context_line":"                    if ts_pref[\u0027timestamp\u0027] in candidate_frag_sets]:"},{"line_number":2483,"context_line":"                available_indexes \u003d [info[\u0027frag_index\u0027]"},{"line_number":2484,"context_line":"                                     for info in candidate_frag_sets[ts]]"},{"line_number":2485,"context_line":"                acceptable_indexes \u003d list(set(available_indexes)"}],"source_content_type":"text/x-python","patch_set":29,"id":"dae33548_8580e2f8","line":2482,"in_reply_to":"3a57f1b5_5451e334","updated":"2016-02-17 08:12:36.000000000","message":"Oh. I noticed I was misreading around _validate_frag_prefs in swift/obj/diskfile.py. I was thinking self._frag_prefs keeps just a json decoded data but _validate_frag_prefs actually translates the value of \u0027timestamp\u0027 from string to Timestamp instance so frag_prefs here will contain Timestamp instances. Therefore I was wrong. Sorry.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":2479,"context_line":"            for ts, ignore_indexes in ["},{"line_number":2480,"context_line":"                    (ts_pref[\u0027timestamp\u0027], ts_pref[\u0027exclude\u0027])"},{"line_number":2481,"context_line":"                    for ts_pref in frag_prefs"},{"line_number":2482,"context_line":"                    if ts_pref[\u0027timestamp\u0027] in candidate_frag_sets]:"},{"line_number":2483,"context_line":"                available_indexes \u003d [info[\u0027frag_index\u0027]"},{"line_number":2484,"context_line":"                                     for info in candidate_frag_sets[ts]]"},{"line_number":2485,"context_line":"                acceptable_indexes \u003d list(set(available_indexes)"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_5451e334","line":2482,"in_reply_to":"5a5ae5dd_bf4fb932","updated":"2016-02-12 18:21:33.000000000","message":"ts_pref[\u0027timestamp\u0027] and the keys in candidate_frag_sets (and frag_sets) *should* all be Timestamp instances - I hope to never be passing around timestamp strings.\n\nDid you find it to be different?","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2500,"context_line":"                # candidate frag_set"},{"line_number":2501,"context_line":"                if candidate_frag_sets:"},{"line_number":2502,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2503,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"},{"line_number":2504,"context_line":"        else:"},{"line_number":2505,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2506,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_7a2cef98","line":2503,"updated":"2016-02-08 10:29:54.000000000","message":"Not sure but should we raise an error to trigger something like 400 Bad request? (or 503, whatever?) Semantically, this statement seems when proxy requested specific fragments (suggested by frag_pref) but not available anything for the request. Am I missing something?\n\nLooking at the proxy code, exactly it seems no matter because if we return the candidates which are not requested, it will overwrite other timestamp\u0027s ResponseBucket as this node is available for them. (I think that info is already set in the first GET request though)\n\nAnd if we return an error code for this, probably we need some changes at bad_bucket in proxy controller, not sure.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d11dc9a684ab8e8658e0e7c4fd6fc71c36f483e","unresolved":false,"context_lines":[{"line_number":2500,"context_line":"                # candidate frag_set"},{"line_number":2501,"context_line":"                if candidate_frag_sets:"},{"line_number":2502,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2503,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"},{"line_number":2504,"context_line":"        else:"},{"line_number":2505,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2506,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"dae33548_700f5988","line":2503,"in_reply_to":"3a57f1b5_274f5b6c","updated":"2016-02-17 08:12:36.000000000","message":"Alright, it seems good. Thanks for explanation :-)","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":2500,"context_line":"                # candidate frag_set"},{"line_number":2501,"context_line":"                if candidate_frag_sets:"},{"line_number":2502,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2503,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"},{"line_number":2504,"context_line":"        else:"},{"line_number":2505,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2506,"context_line":""}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_274f5b6c","line":2503,"in_reply_to":"5a5ae5dd_7a2cef98","updated":"2016-02-12 18:21:33.000000000","message":"The proxy is expressing *preferences* so it is not an error if no preference is found. In fact the proxy could use frag_prefs to prevent the obj server returning anything at a given timestamp (by listing all indexes at that timestamp), which would cause us to get to this point in code.\n\nI will add a comment.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":829,"context_line":"            if exts.get(\u0027.meta\u0027):"},{"line_number":830,"context_line":"                # only report a meta file if there is a data file"},{"line_number":831,"context_line":"                results[\u0027meta_info\u0027] \u003d exts[\u0027.meta\u0027][0]"},{"line_number":832,"context_line":"                ctype_info \u003d exts[\u0027.meta\u0027].pop()"},{"line_number":833,"context_line":"                if (ctype_info[\u0027ctype_timestamp\u0027]"},{"line_number":834,"context_line":"                        \u003e results[\u0027data_info\u0027][\u0027timestamp\u0027]):"},{"line_number":835,"context_line":"                    results[\u0027ctype_info\u0027] \u003d ctype_info"}],"source_content_type":"text/x-python","patch_set":39,"id":"fa7ab95a_5e3c34bf","line":832,"updated":"2016-09-07 12:04:21.000000000","message":"memo) need to get .meta structure. meta_info is using index 0 but ctype_info using the last item of the list... ah, that is needed to resolve retain \u003d 2 case.\n\nOne question still exists. This doesn\u0027t look like related to  the Optimistic GET but it fixes general issue, right? Probably I\u0027m missing something though.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":829,"context_line":"            if exts.get(\u0027.meta\u0027):"},{"line_number":830,"context_line":"                # only report a meta file if there is a data file"},{"line_number":831,"context_line":"                results[\u0027meta_info\u0027] \u003d exts[\u0027.meta\u0027][0]"},{"line_number":832,"context_line":"                ctype_info \u003d exts[\u0027.meta\u0027].pop()"},{"line_number":833,"context_line":"                if (ctype_info[\u0027ctype_timestamp\u0027]"},{"line_number":834,"context_line":"                        \u003e results[\u0027data_info\u0027][\u0027timestamp\u0027]):"},{"line_number":835,"context_line":"                    results[\u0027ctype_info\u0027] \u003d ctype_info"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_1563b8b2","line":832,"in_reply_to":"9a89bdaa_5166c929","updated":"2016-09-14 10:28:27.000000000","message":"Make sense! thanks!","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":829,"context_line":"            if exts.get(\u0027.meta\u0027):"},{"line_number":830,"context_line":"                # only report a meta file if there is a data file"},{"line_number":831,"context_line":"                results[\u0027meta_info\u0027] \u003d exts[\u0027.meta\u0027][0]"},{"line_number":832,"context_line":"                ctype_info \u003d exts[\u0027.meta\u0027].pop()"},{"line_number":833,"context_line":"                if (ctype_info[\u0027ctype_timestamp\u0027]"},{"line_number":834,"context_line":"                        \u003e results[\u0027data_info\u0027][\u0027timestamp\u0027]):"},{"line_number":835,"context_line":"                    results[\u0027ctype_info\u0027] \u003d ctype_info"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_5166c929","line":832,"in_reply_to":"fa7ab95a_5e3c34bf","updated":"2016-09-13 18:49:29.000000000","message":"I think there has been some rebase mistake here - lines 838-844 (or the equivalent code) were moved to 828-835 in an earlier version of the patch, but now lines 838-844 have re-appeared. Will fix.\n\nWhy did the lines move? Previously we would either:\n\n(a) have durable data_info and delete any tombstone that was older from exts, \n\n(b) have no durable and so no data_info\n\nTherefore, previously the elif at line 836 was not necessary.\n\nNow, we may have non-durable data_info, as well as a tombstone that should not be deleted, so we need the elif to ensure we only report the data_info in the results. It does nto make sense to have a data_info and a ts_info in the results, and that would break _verify_ondisk_files.\n\nIf the elif at line 836 is reverted to an if then test_diskfile.py in this patch will fail.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2624,"context_line":"            for t, frag_indexes in [(pref[\u0027timestamp\u0027], pref[\u0027exclude\u0027])"},{"line_number":2625,"context_line":"                                    for pref in frag_prefs]:"},{"line_number":2626,"context_line":"                try:"},{"line_number":2627,"context_line":"                    timestamp \u003d Timestamp(t)"},{"line_number":2628,"context_line":"                except ValueError as e:"},{"line_number":2629,"context_line":"                    raise DiskFileError("},{"line_number":2630,"context_line":"                        \u0027Bad timestamp in frag_prefs: %r: %s\u0027"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_305c284b","line":2627,"updated":"2016-09-07 12:04:21.000000000","message":"Good. We should use Timestamp instance always!","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2624,"context_line":"            for t, frag_indexes in [(pref[\u0027timestamp\u0027], pref[\u0027exclude\u0027])"},{"line_number":2625,"context_line":"                                    for pref in frag_prefs]:"},{"line_number":2626,"context_line":"                try:"},{"line_number":2627,"context_line":"                    timestamp \u003d Timestamp(t)"},{"line_number":2628,"context_line":"                except ValueError as e:"},{"line_number":2629,"context_line":"                    raise DiskFileError("},{"line_number":2630,"context_line":"                        \u0027Bad timestamp in frag_prefs: %r: %s\u0027"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_67b9c02e","line":2627,"in_reply_to":"9a89bdaa_305c284b","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2844,"context_line":"        it is assumed that any fragment index at that timestamp may be used to"},{"line_number":2845,"context_line":"        construct the diskfile."},{"line_number":2846,"context_line":""},{"line_number":2847,"context_line":"        When a frag_prefs arg is provided, including an empty list, there is no"},{"line_number":2848,"context_line":"        requirement for there to be a durable file at the same timestamp as a"},{"line_number":2849,"context_line":"        data file that is chosen to construct the disk file"},{"line_number":2850,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_197c441e","line":2847,"range":{"start_line":2847,"start_character":43,"end_line":2847,"end_character":66},"updated":"2016-09-07 12:04:21.000000000","message":"Yeah, this is why L2888 is \"if frag_prefs is None\". Anytime when object-server got \u0027X-Backend-Fragment-Preferences\u0027 (including empty case), object-server (diskfile actually) return the newest .data file.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2844,"context_line":"        it is assumed that any fragment index at that timestamp may be used to"},{"line_number":2845,"context_line":"        construct the diskfile."},{"line_number":2846,"context_line":""},{"line_number":2847,"context_line":"        When a frag_prefs arg is provided, including an empty list, there is no"},{"line_number":2848,"context_line":"        requirement for there to be a durable file at the same timestamp as a"},{"line_number":2849,"context_line":"        data file that is chosen to construct the disk file"},{"line_number":2850,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_87b6cc3b","line":2847,"range":{"start_line":2847,"start_character":43,"end_line":2847,"end_character":66},"in_reply_to":"9a89bdaa_197c441e","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2890,"context_line":"            # For each per-timestamp frag preference dict, do we have any frag"},{"line_number":2891,"context_line":"            # indexes at that timestamp that are not in the exclusion list for"},{"line_number":2892,"context_line":"            # that timestamp? If so choose the highest of those frag_indexes."},{"line_number":2893,"context_line":"            for ts, ignore_indexes in ["},{"line_number":2894,"context_line":"                    (ts_pref[\u0027timestamp\u0027], ts_pref[\u0027exclude\u0027])"},{"line_number":2895,"context_line":"                    for ts_pref in frag_prefs"},{"line_number":2896,"context_line":"                    if ts_pref[\u0027timestamp\u0027] in candidate_frag_sets]:"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_93825e51","line":2893,"range":{"start_line":2893,"start_character":20,"end_line":2893,"end_character":26},"updated":"2016-09-07 12:04:21.000000000","message":"nits) \"exclude_indexes\" seems better word, maybe/","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2890,"context_line":"            # For each per-timestamp frag preference dict, do we have any frag"},{"line_number":2891,"context_line":"            # indexes at that timestamp that are not in the exclusion list for"},{"line_number":2892,"context_line":"            # that timestamp? If so choose the highest of those frag_indexes."},{"line_number":2893,"context_line":"            for ts, ignore_indexes in ["},{"line_number":2894,"context_line":"                    (ts_pref[\u0027timestamp\u0027], ts_pref[\u0027exclude\u0027])"},{"line_number":2895,"context_line":"                    for ts_pref in frag_prefs"},{"line_number":2896,"context_line":"                    if ts_pref[\u0027timestamp\u0027] in candidate_frag_sets]:"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_8712d307","line":2893,"range":{"start_line":2893,"start_character":20,"end_line":2893,"end_character":26},"in_reply_to":"9a89bdaa_93825e51","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2907,"context_line":"                else:"},{"line_number":2908,"context_line":"                    # this frag_set has no acceptable frag index so"},{"line_number":2909,"context_line":"                    # remove it from the candidate frag_sets"},{"line_number":2910,"context_line":"                    candidate_frag_sets.pop(ts)"},{"line_number":2911,"context_line":"            else:"},{"line_number":2912,"context_line":"                # No acceptable frag index was found at any timestamp mentioned"},{"line_number":2913,"context_line":"                # in the frag_prefs. Choose the newest remaining candidate"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_d335c646","line":2910,"updated":"2016-09-07 12:04:21.000000000","message":"Ah yeah, this is needed at L2916. If no candidate to fit the prefs found but we still other candidates, object-server will report them instead of 404. make sense.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":2907,"context_line":"                else:"},{"line_number":2908,"context_line":"                    # this frag_set has no acceptable frag index so"},{"line_number":2909,"context_line":"                    # remove it from the candidate frag_sets"},{"line_number":2910,"context_line":"                    candidate_frag_sets.pop(ts)"},{"line_number":2911,"context_line":"            else:"},{"line_number":2912,"context_line":"                # No acceptable frag index was found at any timestamp mentioned"},{"line_number":2913,"context_line":"                # in the frag_prefs. Choose the newest remaining candidate"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_75f95cc8","line":2910,"in_reply_to":"9a89bdaa_878653d9","updated":"2016-09-14 10:28:27.000000000","message":"OK, great. Done.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2907,"context_line":"                else:"},{"line_number":2908,"context_line":"                    # this frag_set has no acceptable frag index so"},{"line_number":2909,"context_line":"                    # remove it from the candidate frag_sets"},{"line_number":2910,"context_line":"                    candidate_frag_sets.pop(ts)"},{"line_number":2911,"context_line":"            else:"},{"line_number":2912,"context_line":"                # No acceptable frag index was found at any timestamp mentioned"},{"line_number":2913,"context_line":"                # in the frag_prefs. Choose the newest remaining candidate"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_878653d9","line":2910,"in_reply_to":"9a89bdaa_d335c646","updated":"2016-09-13 18:49:29.000000000","message":"Yes. Remember, the prefs list is exclusive not inclusive, so the semantic is \"report everything other than what the proxy has said it does NOT want\"","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":836,"context_line":"                        \u003e results[\u0027data_info\u0027][\u0027timestamp\u0027]):"},{"line_number":837,"context_line":"                    results[\u0027ctype_info\u0027] \u003d ctype_info"},{"line_number":838,"context_line":"        elif exts.get(\u0027.ts\u0027):"},{"line_number":839,"context_line":"            # only report a ts file if a data file has not been chosen"},{"line_number":840,"context_line":"            # (ts files will commonly already have been removed from exts if"},{"line_number":841,"context_line":"            # a data file was chosen, but that may not be the case if"},{"line_number":842,"context_line":"            # non-durable EC fragment(s) were chosen, hence the elif here)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_f5ed6c8f","line":839,"updated":"2016-09-14 03:15:04.000000000","message":"Am I correct in thinking that previously, either a data file was chosen or a ts remained, but never both?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":836,"context_line":"                        \u003e results[\u0027data_info\u0027][\u0027timestamp\u0027]):"},{"line_number":837,"context_line":"                    results[\u0027ctype_info\u0027] \u003d ctype_info"},{"line_number":838,"context_line":"        elif exts.get(\u0027.ts\u0027):"},{"line_number":839,"context_line":"            # only report a ts file if a data file has not been chosen"},{"line_number":840,"context_line":"            # (ts files will commonly already have been removed from exts if"},{"line_number":841,"context_line":"            # a data file was chosen, but that may not be the case if"},{"line_number":842,"context_line":"            # non-durable EC fragment(s) were chosen, hence the elif here)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_f62f083c","line":839,"in_reply_to":"7a8ec9b2_f5ed6c8f","updated":"2016-09-14 17:32:23.000000000","message":"yes. the previous case is what the following comment refers to as the common case. Having a durable data file caused any older ts to be removed, having a newer ts causes data files to be removed.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":850,"context_line":"            results[key] \u003d join(datadir, info[\u0027filename\u0027]) if info else None"},{"line_number":851,"context_line":""},{"line_number":852,"context_line":"        if verify:"},{"line_number":853,"context_line":"            assert self._verify_ondisk_files("},{"line_number":854,"context_line":"                results, **kwargs), \\"},{"line_number":855,"context_line":"                \"On-disk file search algorithm contract is broken: %s\" \\"},{"line_number":856,"context_line":"                % str(results)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_b22f7aa0","line":853,"updated":"2016-09-14 03:15:04.000000000","message":"Are we assuming that no one runs with optimized code?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f0b426ce42cce4d4f9248a5d22b81cd931d9b9b4","unresolved":false,"context_lines":[{"line_number":850,"context_line":"            results[key] \u003d join(datadir, info[\u0027filename\u0027]) if info else None"},{"line_number":851,"context_line":""},{"line_number":852,"context_line":"        if verify:"},{"line_number":853,"context_line":"            assert self._verify_ondisk_files("},{"line_number":854,"context_line":"                results, **kwargs), \\"},{"line_number":855,"context_line":"                \"On-disk file search algorithm contract is broken: %s\" \\"},{"line_number":856,"context_line":"                % str(results)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_22091bde","line":853,"in_reply_to":"7a8ec9b2_b22f7aa0","updated":"2016-09-14 19:53:06.000000000","message":"someone added an assert into the code one time because the code was complex and seemed easy to get wrong - the idea that we would rather break it earlier than later made sense and we let it in along with a big refactor\n\nlater we realized we hate asserts in the code and would very much rather this sort of thing be handled with sufficient tests - which it is - but no one managed to rip it out\n\nsomeone should! [1]\n\n1. unless no one does - then the assert either works and no one cares or it doesn\u0027t run and no one notices","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":2678,"context_line":"        except (KeyError, TypeError) as e:"},{"line_number":2679,"context_line":"            raise DiskFileError("},{"line_number":2680,"context_line":"                \u0027Bad frag_prefs: %r: %s\u0027 % (frag_prefs, e))"},{"line_number":2681,"context_line":"        return result"},{"line_number":2682,"context_line":""},{"line_number":2683,"context_line":"    @property"},{"line_number":2684,"context_line":"    def durable_timestamp(self):"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_55e2a004","line":2681,"updated":"2016-09-14 03:15:04.000000000","message":"Maybe better as\n\n try:\n     return [\n         {\u0027timestamp\u0027: Timestamp(pref[\u0027timestamp\u0027]),\n          \u0027exclude\u0027: [self.manager.validate_fragment_index(fi)\n                      for fi in pref[\u0027exclude\u0027]]}\n         for pref in frag_prefs]\n except ValueError as e:\n     raise DiskFileError(\n         \u0027Bad timestamp in frag_prefs: %r: %s\u0027\n         % (frag_prefs, e))\n except DiskFileError as e:\n     raise DiskFileError(\n         \u0027Bad fragment index in frag_prefs: %r: %s\u0027\n         % (frag_prefs, e))\n except (KeyError, TypeError) as e:\n     raise DiskFileError(\n         \u0027Bad frag_prefs: %r: %s\u0027 % (frag_prefs, e))\n\n?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":2678,"context_line":"        except (KeyError, TypeError) as e:"},{"line_number":2679,"context_line":"            raise DiskFileError("},{"line_number":2680,"context_line":"                \u0027Bad frag_prefs: %r: %s\u0027 % (frag_prefs, e))"},{"line_number":2681,"context_line":"        return result"},{"line_number":2682,"context_line":""},{"line_number":2683,"context_line":"    @property"},{"line_number":2684,"context_line":"    def durable_timestamp(self):"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_5ce8dde2","line":2681,"in_reply_to":"7a8ec9b2_55e2a004","updated":"2016-09-14 17:32:23.000000000","message":"Done, thanks","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":2950,"context_line":"                # No acceptable frag index was found at any timestamp mentioned"},{"line_number":2951,"context_line":"                # in the frag_prefs. Choose the newest remaining candidate"},{"line_number":2952,"context_line":"                # frag_set - the proxy can decide if it wants the returned"},{"line_number":2953,"context_line":"                # fragment with that time."},{"line_number":2954,"context_line":"                if candidate_frag_sets:"},{"line_number":2955,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2956,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_80a06492","line":2953,"updated":"2016-09-14 03:15:04.000000000","message":"I feel like this behavior is only really appropriate if len(frag_prefs) \u003d\u003d 0 -- i.e. the proxy has no idea what\u0027s out there. Once the proxy know enough to say, \"hey, these are of interest,\" is there any chance that it\u0027ll go back to something not explicitly in frag_prefs?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":2950,"context_line":"                # No acceptable frag index was found at any timestamp mentioned"},{"line_number":2951,"context_line":"                # in the frag_prefs. Choose the newest remaining candidate"},{"line_number":2952,"context_line":"                # frag_set - the proxy can decide if it wants the returned"},{"line_number":2953,"context_line":"                # fragment with that time."},{"line_number":2954,"context_line":"                if candidate_frag_sets:"},{"line_number":2955,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2956,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_4a72cbae","line":2953,"in_reply_to":"7a8ec9b2_80a06492","updated":"2016-09-14 17:32:23.000000000","message":"with the caveat that it has been months since I last reasoned over this...the proxy will often not yet know of all possible timestamps when setting the frag prefs - bear in mind that the bucket collection is queried for frag prefs on every request so even in the first cohort of concurrent ec_ndata requests, some that are on \"slower\" getter threads might get a frag pref setting based on an early response on a \"faster\" getter thread.\n\nSo while the frag prefs does express \"these timestamps are of interest\" it should not convey \"and no others\".\n\nTake an example: primaries have insufficient frags at t1 and no other frags. handoffs have a complete set of frags at t2. first set of requests result in a shortfall and frag_prefs gets populated with t1, then requests go to handoffs - we want them to report back with their frags at t2.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f0b426ce42cce4d4f9248a5d22b81cd931d9b9b4","unresolved":false,"context_lines":[{"line_number":2955,"context_line":"                    ts_newest \u003d sorted(candidate_frag_sets.keys())[-1]"},{"line_number":2956,"context_line":"                    chosen_frag_set \u003d candidate_frag_sets[ts_newest]"},{"line_number":2957,"context_line":"        else:"},{"line_number":2958,"context_line":"            chosen_frag_set \u003d durable_frag_set"},{"line_number":2959,"context_line":""},{"line_number":2960,"context_line":"        # Select a single chosen frag from the chosen frag_set, by either"},{"line_number":2961,"context_line":"        # matching against a specified frag_index or taking the highest index."}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_c2f237d2","line":2958,"updated":"2016-09-14 19:53:06.000000000","message":"... and our optimism only ever kicks in if it\u0027s requested!","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":3002,"context_line":"    def _verify_ondisk_files(self, results, frag_index\u003dNone,"},{"line_number":3003,"context_line":"                             frag_prefs\u003dNone, **kwargs):"},{"line_number":3004,"context_line":"        \"\"\""},{"line_number":3005,"context_line":"        Verify that the final combination of on disk files complies with the"},{"line_number":3006,"context_line":"        erasure-coded diskfile contract."},{"line_number":3007,"context_line":""},{"line_number":3008,"context_line":"        :param results: files that have been found and accepted"},{"line_number":3009,"context_line":"        :param frag_index: specifies a specific fragment index .data file"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_52e2b604","line":3006,"range":{"start_line":3005,"start_character":59,"end_line":3006,"end_character":39},"updated":"2016-09-14 03:15:04.000000000","message":"Where is this contract defined? Is this a contract that can be reasoned about locally (on a single node), or does it only make sense when we\u0027re considering a collection of nodes?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":3002,"context_line":"    def _verify_ondisk_files(self, results, frag_index\u003dNone,"},{"line_number":3003,"context_line":"                             frag_prefs\u003dNone, **kwargs):"},{"line_number":3004,"context_line":"        \"\"\""},{"line_number":3005,"context_line":"        Verify that the final combination of on disk files complies with the"},{"line_number":3006,"context_line":"        erasure-coded diskfile contract."},{"line_number":3007,"context_line":""},{"line_number":3008,"context_line":"        :param results: files that have been found and accepted"},{"line_number":3009,"context_line":"        :param frag_index: specifies a specific fragment index .data file"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_eaa1d7aa","line":3006,"range":{"start_line":3005,"start_character":59,"end_line":3006,"end_character":39},"in_reply_to":"7a8ec9b2_52e2b604","updated":"2016-09-14 17:32:23.000000000","message":"It is defined by this method, and it applies locally (in scope of diskfile).","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":3008,"context_line":"        :param results: files that have been found and accepted"},{"line_number":3009,"context_line":"        :param frag_index: specifies a specific fragment index .data file"},{"line_number":3010,"context_line":"        :param frag_prefs: if set, search for any fragment index .data file"},{"line_number":3011,"context_line":"                           that satisfies the frag_prefs."},{"line_number":3012,"context_line":"        :returns: True if the file combination is compliant, False otherwise"},{"line_number":3013,"context_line":"        \"\"\""},{"line_number":3014,"context_line":"        if super(ECDiskFileManager, self)._verify_ondisk_files("}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_0a30c39d","line":3011,"updated":"2016-09-14 17:32:23.000000000","message":"param definition is not appropriately worded","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":3016,"context_line":"            have_data_file \u003d results[\u0027data_file\u0027] is not None"},{"line_number":3017,"context_line":"            have_durable \u003d (results.get(\u0027durable_frag_set\u0027) is not None or"},{"line_number":3018,"context_line":"                            (have_data_file and frag_prefs is not None))"},{"line_number":3019,"context_line":"            return have_data_file \u003d\u003d have_durable"},{"line_number":3020,"context_line":"        return False"},{"line_number":3021,"context_line":""},{"line_number":3022,"context_line":"    def _update_suffix_hashes(self, hashes, ondisk_info):"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_d29086ef","line":3019,"updated":"2016-09-14 03:15:04.000000000","message":"Ugh, that makes for an ugly truth-table :-/","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"swift/obj/server.py":[{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f832c2009aa370f89e75db03e1e32d099d9511d","unresolved":false,"context_lines":[{"line_number":719,"context_line":"        device, partition, account, container, obj, policy \u003d \\"},{"line_number":720,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":721,"context_line":"        require_durable \u003d config_true_value("},{"line_number":722,"context_line":"            request.headers.get(\u0027X-Require-Durable\u0027, True))"},{"line_number":723,"context_line":"        try:"},{"line_number":724,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":725,"context_line":"                device, partition, account, container, obj,"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_716a361c","line":722,"updated":"2015-09-15 19:18:49.000000000","message":"make this a Backend header","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1ed91bacb04552b64455cf813f0d4515383b0d24","unresolved":false,"context_lines":[{"line_number":719,"context_line":"        device, partition, account, container, obj, policy \u003d \\"},{"line_number":720,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":721,"context_line":"        require_durable \u003d config_true_value("},{"line_number":722,"context_line":"            request.headers.get(\u0027X-Require-Durable\u0027, True))"},{"line_number":723,"context_line":"        try:"},{"line_number":724,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":725,"context_line":"                device, partition, account, container, obj,"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_f4984dc2","line":722,"in_reply_to":"ba15a1d1_716a361c","updated":"2015-09-16 19:11:17.000000000","message":"Done","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"4f832c2009aa370f89e75db03e1e32d099d9511d","unresolved":false,"context_lines":[{"line_number":758,"context_line":"                    pass"},{"line_number":759,"context_line":"                response.headers[\u0027X-Timestamp\u0027] \u003d file_x_ts.normal"},{"line_number":760,"context_line":"                response.headers[\u0027X-Backend-Timestamp\u0027] \u003d file_x_ts.internal"},{"line_number":761,"context_line":"                response.headers[\u0027X-Durable\u0027] \u003d str(disk_file.is_durable())"},{"line_number":762,"context_line":"                resp \u003d request.get_response(response)"},{"line_number":763,"context_line":"        except DiskFileXattrNotSupported:"},{"line_number":764,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_d161c23a","line":761,"updated":"2015-09-15 19:18:49.000000000","message":"make this a Backend header","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1ed91bacb04552b64455cf813f0d4515383b0d24","unresolved":false,"context_lines":[{"line_number":758,"context_line":"                    pass"},{"line_number":759,"context_line":"                response.headers[\u0027X-Timestamp\u0027] \u003d file_x_ts.normal"},{"line_number":760,"context_line":"                response.headers[\u0027X-Backend-Timestamp\u0027] \u003d file_x_ts.internal"},{"line_number":761,"context_line":"                response.headers[\u0027X-Durable\u0027] \u003d str(disk_file.is_durable())"},{"line_number":762,"context_line":"                resp \u003d request.get_response(response)"},{"line_number":763,"context_line":"        except DiskFileXattrNotSupported:"},{"line_number":764,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_347a4578","line":761,"in_reply_to":"ba15a1d1_d161c23a","updated":"2015-09-16 19:11:17.000000000","message":"Done","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"41f92a2bcc61186ffc4909fdb80dd98695d8703f","unresolved":false,"context_lines":[{"line_number":779,"context_line":"        try:"},{"line_number":780,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":781,"context_line":"                device, partition, account, container, obj,"},{"line_number":782,"context_line":"                policy\u003dpolicy, require_durable\u003dTrue)"},{"line_number":783,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":784,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":785,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"da20952f_b43f22de","line":782,"updated":"2015-09-11 17:12:30.000000000","message":"Should we make HEADs also repsect X-Require-Durable in order to be truly RESTful ie always return same headers for a HEAD as  GET? - as it is HEAD will always return X-Durable\u003dTrue.","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"1ed91bacb04552b64455cf813f0d4515383b0d24","unresolved":false,"context_lines":[{"line_number":779,"context_line":"        try:"},{"line_number":780,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":781,"context_line":"                device, partition, account, container, obj,"},{"line_number":782,"context_line":"                policy\u003dpolicy, require_durable\u003dTrue)"},{"line_number":783,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":784,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":785,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_39f1e001","line":782,"in_reply_to":"ba15a1d1_4d93d2a3","updated":"2015-09-16 19:11:17.000000000","message":"Done","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ec1575a884c0a49ffb9b27f17a4dff1e5794ba44","unresolved":false,"context_lines":[{"line_number":779,"context_line":"        try:"},{"line_number":780,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":781,"context_line":"                device, partition, account, container, obj,"},{"line_number":782,"context_line":"                policy\u003dpolicy, require_durable\u003dTrue)"},{"line_number":783,"context_line":"        except DiskFileDeviceUnavailable:"},{"line_number":784,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":785,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_4d93d2a3","line":782,"in_reply_to":"da20952f_b43f22de","updated":"2015-09-15 22:45:13.000000000","message":"yeah - it should do the same as GET - what\u0027s the counter argument?","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":719,"context_line":"        device, partition, account, container, obj, policy \u003d \\"},{"line_number":720,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":721,"context_line":"        require_durable \u003d config_true_value("},{"line_number":722,"context_line":"            request.headers.get(\u0027X-Backend-Require-Durable\u0027, True))"},{"line_number":723,"context_line":"        try:"},{"line_number":724,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":725,"context_line":"                device, partition, account, container, obj,"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_0bfe1838","line":722,"updated":"2015-09-22 22:00:39.000000000","message":"so the header would have to be there and explicitly set to false","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":719,"context_line":"        device, partition, account, container, obj, policy \u003d \\"},{"line_number":720,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":721,"context_line":"        require_durable \u003d config_true_value("},{"line_number":722,"context_line":"            request.headers.get(\u0027X-Backend-Require-Durable\u0027, True))"},{"line_number":723,"context_line":"        try:"},{"line_number":724,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":725,"context_line":"                device, partition, account, container, obj,"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_d342ea0e","line":722,"in_reply_to":"ba15a1d1_0bfe1838","updated":"2015-09-23 20:03:04.000000000","message":"yes. so if an not-upgraded proxy sends a GET we\u0027ll always return a durable frag","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":761,"context_line":"                response.headers[\u0027X-Backend-Durable\u0027] \u003d disk_file.is_durable()"},{"line_number":762,"context_line":"                if disk_file.durable_timestamp:"},{"line_number":763,"context_line":"                    response.headers[\u0027X-Backend-Durable-Timestamp\u0027] \u003d \\"},{"line_number":764,"context_line":"                        disk_file.durable_timestamp.internal"},{"line_number":765,"context_line":"                resp \u003d request.get_response(response)"},{"line_number":766,"context_line":"        except DiskFileXattrNotSupported:"},{"line_number":767,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_2b9f5c04","line":764,"updated":"2015-09-22 22:00:39.000000000","message":"Seems like it would again be useful to just have X-Backend-Durable-Timestamp - does the replicated case ever look at X-Backend-Durable?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":761,"context_line":"                response.headers[\u0027X-Backend-Durable\u0027] \u003d disk_file.is_durable()"},{"line_number":762,"context_line":"                if disk_file.durable_timestamp:"},{"line_number":763,"context_line":"                    response.headers[\u0027X-Backend-Durable-Timestamp\u0027] \u003d \\"},{"line_number":764,"context_line":"                        disk_file.durable_timestamp.internal"},{"line_number":765,"context_line":"                resp \u003d request.get_response(response)"},{"line_number":766,"context_line":"        except DiskFileXattrNotSupported:"},{"line_number":767,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_f32cce4b","line":764,"in_reply_to":"ba15a1d1_2b9f5c04","updated":"2015-09-23 20:03:04.000000000","message":"I need a way to distinguish between (a) old obj server that sets none of these new headers \u003d\u003d durable and (b) new obj server that has no durable timestamp \u003d\u003d not durable.\n\nI found a different way in next version so the X-Backend-Durable is not needed, just timestamps.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":82,"context_line":"                break"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"def _encode_backend_fragments(fragments):"},{"line_number":86,"context_line":"    if fragments:"},{"line_number":87,"context_line":"        result \u003d {}"},{"line_number":88,"context_line":"        for ts, frag_list in fragments.items():"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_58c33489","line":85,"updated":"2015-09-30 12:56:04.000000000","message":"Strictly, it seems \"_encode_backend_fragments_dict\"?\n\nI\u0027d like to change this function name because \"encode fragments\" seems confusing for readers like as \"Does this function encode incoming data?\" but not true.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":82,"context_line":"                break"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"def _encode_backend_fragments(fragments):"},{"line_number":86,"context_line":"    if fragments:"},{"line_number":87,"context_line":"        result \u003d {}"},{"line_number":88,"context_line":"        for ts, frag_list in fragments.items():"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_44f9787f","line":85,"in_reply_to":"9a1a9d01_55ca121c","updated":"2015-10-05 07:08:59.000000000","message":"Nice, that\u0027s intuitive for us :)","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":82,"context_line":"                break"},{"line_number":83,"context_line":""},{"line_number":84,"context_line":""},{"line_number":85,"context_line":"def _encode_backend_fragments(fragments):"},{"line_number":86,"context_line":"    if fragments:"},{"line_number":87,"context_line":"        result \u003d {}"},{"line_number":88,"context_line":"        for ts, frag_list in fragments.items():"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_55ca121c","line":85,"in_reply_to":"9a1a9d01_58c33489","updated":"2015-09-30 23:33:45.000000000","message":"I see what you are saying.\n\nHow about _encode_diskfile_fragment_info ?","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"f0b426ce42cce4d4f9248a5d22b81cd931d9b9b4","unresolved":false,"context_lines":[{"line_number":863,"context_line":"        device, partition, account, container, obj, policy \u003d \\"},{"line_number":864,"context_line":"            get_name_and_placement(request, 5, 5, True)"},{"line_number":865,"context_line":"        frag_prefs \u003d safe_json_loads("},{"line_number":866,"context_line":"            request.headers.get(\u0027X-Backend-Fragment-Preferences\u0027))"},{"line_number":867,"context_line":"        try:"},{"line_number":868,"context_line":"            disk_file \u003d self.get_diskfile("},{"line_number":869,"context_line":"                device, partition, account, container, obj,"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_4258e732","line":866,"updated":"2016-09-14 19:53:06.000000000","message":"so the optimism starts here - from the request param","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"swift/proxy/controllers/base.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":981,"context_line":"                continue"},{"line_number":982,"context_line":"            start_node_timing \u003d time.time()"},{"line_number":983,"context_line":"            # a node may be specialised with specific backend headers"},{"line_number":984,"context_line":"            self.backend_headers.update(node.get(\u0027headers\u0027, {}))"},{"line_number":985,"context_line":"            try:"},{"line_number":986,"context_line":"                with ConnectionTimeout(self.app.conn_timeout):"},{"line_number":987,"context_line":"                    conn \u003d http_connect("}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ae813a91","line":984,"updated":"2015-09-22 22:00:39.000000000","message":"oh goodness :\\\n\ni missed that in the ObjectController code","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1226,"context_line":"        # TODO we may prefer to return alt nodes only after all primary nodes"},{"line_number":1227,"context_line":"        # have been exhausted, in which case we\u0027ll need to split"},{"line_number":1228,"context_line":"        # self._node_iter into two iters and insert the alt-nodes before the"},{"line_number":1229,"context_line":"        # handoffs."},{"line_number":1230,"context_line":"        # TODO check this is ok wrt the python iterator contract [1] - this"},{"line_number":1231,"context_line":"        # may return a node after StopIteration has been previously raised."},{"line_number":1232,"context_line":"        # [1] https://docs.python.org/2/library/stdtypes.html#iterator-types"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_093620e7","line":1229,"updated":"2015-09-22 22:00:39.000000000","message":"yeah that\u0027s probably what we want :\\\n\nI mean it seems like we could just stick it in _node_gen?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1229,"context_line":"        # handoffs."},{"line_number":1230,"context_line":"        # TODO check this is ok wrt the python iterator contract [1] - this"},{"line_number":1231,"context_line":"        # may return a node after StopIteration has been previously raised."},{"line_number":1232,"context_line":"        # [1] https://docs.python.org/2/library/stdtypes.html#iterator-types"},{"line_number":1233,"context_line":"        try:"},{"line_number":1234,"context_line":"            return self.alt_nodes.pop(0)"},{"line_number":1235,"context_line":"        except IndexError:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_2920c49a","line":1232,"updated":"2015-09-22 22:00:39.000000000","message":"yeah I mean I think that\u0027s just how piles work?\n\n  \u003e\u003e\u003e from eventlet import GreenPile, sleep\n  \u003e\u003e\u003e p \u003d GreenPile()\n  \u003e\u003e\u003e p.spawn(sleep, 1)\n  \u003e\u003e\u003e i \u003d iter(p)\n  \u003e\u003e\u003e next(i)\n  \u003e\u003e\u003e next(i)\n  Traceback (most recent call last):\n    File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\n    File \"/usr/local/lib/python2.7/dist-packages/eventlet/greenpool.py\", line 220, in next\n      raise StopIteration()\n  StopIteration\n  \u003e\u003e\u003e p.spawn(sleep, 1)\n  \u003e\u003e\u003e next(i)\n  \u003e\u003e\u003e next(i)\n  Traceback (most recent call last):\n    File \"\u003cstdin\u003e\", line 1, in \u003cmodule\u003e\n    File \"/usr/local/lib/python2.7/dist-packages/eventlet/greenpool.py\", line 220, in next\n      raise StopIteration()\n  StopIteration","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":1229,"context_line":"        # handoffs."},{"line_number":1230,"context_line":"        # TODO check this is ok wrt the python iterator contract [1] - this"},{"line_number":1231,"context_line":"        # may return a node after StopIteration has been previously raised."},{"line_number":1232,"context_line":"        # [1] https://docs.python.org/2/library/stdtypes.html#iterator-types"},{"line_number":1233,"context_line":"        try:"},{"line_number":1234,"context_line":"            return self.alt_nodes.pop(0)"},{"line_number":1235,"context_line":"        except IndexError:"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_fe9b35c7","line":1232,"in_reply_to":"ba15a1d1_2920c49a","updated":"2015-09-23 20:03:04.000000000","message":"thanks!","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1204,"context_line":"            self.used_source_etag \u003d src_headers.get("},{"line_number":1205,"context_line":"                \u0027x-object-sysmeta-ec-etag\u0027,"},{"line_number":1206,"context_line":"                src_headers.get(\u0027etag\u0027, \u0027\u0027)).strip(\u0027\"\u0027)"},{"line_number":1207,"context_line":"            self.node \u003d node"},{"line_number":1208,"context_line":"            return source, node"},{"line_number":1209,"context_line":"        return None, None"},{"line_number":1210,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_1ce57b08","line":1207,"updated":"2016-09-07 12:04:21.000000000","message":"memo) okay, this is needed to set the available node getting  via resuming getter as an alternate node at swift/proxy/controllers/obj.py.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1204,"context_line":"            self.used_source_etag \u003d src_headers.get("},{"line_number":1205,"context_line":"                \u0027x-object-sysmeta-ec-etag\u0027,"},{"line_number":1206,"context_line":"                src_headers.get(\u0027etag\u0027, \u0027\u0027)).strip(\u0027\"\u0027)"},{"line_number":1207,"context_line":"            self.node \u003d node"},{"line_number":1208,"context_line":"            return source, node"},{"line_number":1209,"context_line":"        return None, None"},{"line_number":1210,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_c7d69bc2","line":1207,"in_reply_to":"9a89bdaa_1ce57b08","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"}],"swift/proxy/controllers/obj.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1879,"context_line":"    \"\"\""},{"line_number":1880,"context_line":"    A helper class to encapsulate the properties of buckets in which fragment"},{"line_number":1881,"context_line":"    getters and laternate nodes are collected."},{"line_number":1882,"context_line":"    \"\"\""},{"line_number":1883,"context_line":"    def __init__(self, quorum):"},{"line_number":1884,"context_line":"        self.quorum \u003d quorum"},{"line_number":1885,"context_line":"        self.gets \u003d []"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_eb7724d2","line":1882,"updated":"2015-09-22 22:00:39.000000000","message":"nice.  ResposneBucket, ObjectResponseBucket\n\nI wonder if this was an ECResponseCollection if we could just feed responses and pull out some of the questions in _get_or_head_response like `best_bucket` and `[total_]shortfall`","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":1879,"context_line":"    \"\"\""},{"line_number":1880,"context_line":"    A helper class to encapsulate the properties of buckets in which fragment"},{"line_number":1881,"context_line":"    getters and laternate nodes are collected."},{"line_number":1882,"context_line":"    \"\"\""},{"line_number":1883,"context_line":"    def __init__(self, quorum):"},{"line_number":1884,"context_line":"        self.quorum \u003d quorum"},{"line_number":1885,"context_line":"        self.gets \u003d []"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_53a49a49","line":1882,"in_reply_to":"ba15a1d1_eb7724d2","updated":"2015-09-23 20:03:04.000000000","message":"Done: change class name\n\nack Collection idea: super idea, will add to wish list","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"            # we can get more specific e.g. specify exactly which frag index at"},{"line_number":1916,"context_line":"            # which timestamp. That would also allow us to get older"},{"line_number":1917,"context_line":"            # non-durables."},{"line_number":1918,"context_line":"            alt_node[\u0027headers\u0027] \u003d {\u0027X-Backend-Require-Durable\u0027: \u0027True\u0027}"},{"line_number":1919,"context_line":"            result.append(alt_node)"},{"line_number":1920,"context_line":"        self.alt_nodes \u003d []"},{"line_number":1921,"context_line":"        return result"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_e9af2ce1","line":1918,"updated":"2015-09-22 22:00:39.000000000","message":"ok, so this is the bit where we use the node dicts to add headers for their subsequent requests :\\","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":1915,"context_line":"            # we can get more specific e.g. specify exactly which frag index at"},{"line_number":1916,"context_line":"            # which timestamp. That would also allow us to get older"},{"line_number":1917,"context_line":"            # non-durables."},{"line_number":1918,"context_line":"            alt_node[\u0027headers\u0027] \u003d {\u0027X-Backend-Require-Durable\u0027: \u0027True\u0027}"},{"line_number":1919,"context_line":"            result.append(alt_node)"},{"line_number":1920,"context_line":"        self.alt_nodes \u003d []"},{"line_number":1921,"context_line":"        return result"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_33732650","line":1918,"in_reply_to":"ba15a1d1_e9af2ce1","updated":"2015-09-23 20:03:04.000000000","message":"Yeah. Any better ideas? Is this too hokey?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":1933,"context_line":"    def shortfall(self):"},{"line_number":1934,"context_line":"        # a non-durable bucket always has a shortfall of at least 1"},{"line_number":1935,"context_line":"        result \u003d self.quorum - len(self.gets)"},{"line_number":1936,"context_line":"        if not self.durable and result \u003c 1:"},{"line_number":1937,"context_line":"            result \u003d 1"},{"line_number":1938,"context_line":"        return result"},{"line_number":1939,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ee13b25a","line":1936,"updated":"2015-09-22 22:00:39.000000000","message":"i\u0027m not sure order of operations agrees with the above comment?\n\n    if (not self.durable) and (result \u003c 1):\n\n... oh i guess that works, still maybe\n\n    if result \u003c 1 and not self.durable:\n\n... or\n\n    return max(result, 0 if self.durable else 1)","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":1933,"context_line":"    def shortfall(self):"},{"line_number":1934,"context_line":"        # a non-durable bucket always has a shortfall of at least 1"},{"line_number":1935,"context_line":"        result \u003d self.quorum - len(self.gets)"},{"line_number":1936,"context_line":"        if not self.durable and result \u003c 1:"},{"line_number":1937,"context_line":"            result \u003d 1"},{"line_number":1938,"context_line":"        return result"},{"line_number":1939,"context_line":""}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_93cf9255","line":1936,"in_reply_to":"ba15a1d1_ee13b25a","updated":"2015-09-23 20:03:04.000000000","message":"Done","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2075,"context_line":"            bad_bucket \u003d Bucket(policy.ec_ndata)"},{"line_number":2076,"context_line":"            buckets \u003d collections.defaultdict("},{"line_number":2077,"context_line":"                functools.partial(Bucket, policy.ec_ndata))"},{"line_number":2078,"context_line":"            best_bucket \u003d Bucket(policy.ec_ndata)"},{"line_number":2079,"context_line":"            for get, parts_iter in pile:"},{"line_number":2080,"context_line":"                if get.last_status is None:"},{"line_number":2081,"context_line":"                    # We may have spawned getters that find the node iterator"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_cef7ce57","line":2078,"updated":"2015-09-22 22:00:39.000000000","message":"maybe, not sure this is better than\n\n    best_bucket \u003d None\n    ....\n    if best_bucket and best_bucket.shortfall \u003c\u003d 0:\n        ...","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2075,"context_line":"            bad_bucket \u003d Bucket(policy.ec_ndata)"},{"line_number":2076,"context_line":"            buckets \u003d collections.defaultdict("},{"line_number":2077,"context_line":"                functools.partial(Bucket, policy.ec_ndata))"},{"line_number":2078,"context_line":"            best_bucket \u003d Bucket(policy.ec_ndata)"},{"line_number":2079,"context_line":"            for get, parts_iter in pile:"},{"line_number":2080,"context_line":"                if get.last_status is None:"},{"line_number":2081,"context_line":"                    # We may have spawned getters that find the node iterator"}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_f3604e37","line":2078,"in_reply_to":"ba15a1d1_cef7ce57","updated":"2015-09-23 20:03:04.000000000","message":"ack, but I\u0027d need \u0027if best_bucket\u0027 in several places. Maybe this will get cleaner if I run with your collection idea.","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"9e125684b61b9939e509e41363bdbe441beab4f1","unresolved":false,"context_lines":[{"line_number":2086,"context_line":"                    # of None. We should fix that but keep this guard because"},{"line_number":2087,"context_line":"                    # there is also a race between testing nodes_left/spawning"},{"line_number":2088,"context_line":"                    # a getter and an existing getter calling next(node_iter)."},{"line_number":2089,"context_line":"                    continue"},{"line_number":2090,"context_line":"                if is_success(get.last_status):"},{"line_number":2091,"context_line":"                    # all 2xx responses are placed in bucket keyed with a"},{"line_number":2092,"context_line":"                    # timestamp string."}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_ee541255","line":2089,"updated":"2015-09-22 22:00:39.000000000","message":"bah, maybe get rid of nodes_left?","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"a963a5bc8d9f38832c44ac0937a7f6512196fec8","unresolved":false,"context_lines":[{"line_number":2086,"context_line":"                    # of None. We should fix that but keep this guard because"},{"line_number":2087,"context_line":"                    # there is also a race between testing nodes_left/spawning"},{"line_number":2088,"context_line":"                    # a getter and an existing getter calling next(node_iter)."},{"line_number":2089,"context_line":"                    continue"},{"line_number":2090,"context_line":"                if is_success(get.last_status):"},{"line_number":2091,"context_line":"                    # all 2xx responses are placed in bucket keyed with a"},{"line_number":2092,"context_line":"                    # timestamp string."}],"source_content_type":"text/x-python","patch_set":8,"id":"ba15a1d1_93523266","line":2089,"in_reply_to":"ba15a1d1_ee541255","updated":"2015-09-23 20:03:04.000000000","message":"ack, but can I leave that for a follow on? I have a feeling it won\u0027t be really simple - getting a None status back does not necessarily mean the NodeIter is drained because we may have added more nodes (below).","commit_id":"2d5f4e7f9ceed5df0aa71b3932e9a20c483700c3"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":1905,"context_line":"    getters and alternate nodes are collected."},{"line_number":1906,"context_line":"    \"\"\""},{"line_number":1907,"context_line":"    def __init__(self, quorum, timestamp):"},{"line_number":1908,"context_line":"        self.quorum \u003d quorum"},{"line_number":1909,"context_line":"        self.timestamp \u003d timestamp"},{"line_number":1910,"context_line":"        self.gets \u003d collections.defaultdict(list)"},{"line_number":1911,"context_line":"        self.alt_nodes \u003d collections.defaultdict(list)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_8e00a667","line":1908,"updated":"2015-09-30 12:56:04.000000000","message":"Is this self.ec_ndata? See my comment at L1999 in detail.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":1905,"context_line":"    getters and alternate nodes are collected."},{"line_number":1906,"context_line":"    \"\"\""},{"line_number":1907,"context_line":"    def __init__(self, quorum, timestamp):"},{"line_number":1908,"context_line":"        self.quorum \u003d quorum"},{"line_number":1909,"context_line":"        self.timestamp \u003d timestamp"},{"line_number":1910,"context_line":"        self.gets \u003d collections.defaultdict(list)"},{"line_number":1911,"context_line":"        self.alt_nodes \u003d collections.defaultdict(list)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_509ea0b9","line":1908,"in_reply_to":"9a1a9d01_8e00a667","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":1962,"context_line":"        result \u003d self.quorum - (len(self.get_responses())"},{"line_number":1963,"context_line":"                                + len(alts))"},{"line_number":1964,"context_line":"        return max(result, 0 if self._durable else 1)"},{"line_number":1965,"context_line":""},{"line_number":1966,"context_line":"    def __str__(self):"},{"line_number":1967,"context_line":"        return \u0027\u003c%s, %s, %s, %s(%s), %s\u003e\u0027 \\"},{"line_number":1968,"context_line":"               % (self.timestamp, self.status, self._durable, self.shortfall,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_bb15aa9c","line":1965,"updated":"2015-09-30 12:56:04.000000000","message":"(memo) to see the usage.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":1962,"context_line":"        result \u003d self.quorum - (len(self.get_responses())"},{"line_number":1963,"context_line":"                                + len(alts))"},{"line_number":1964,"context_line":"        return max(result, 0 if self._durable else 1)"},{"line_number":1965,"context_line":""},{"line_number":1966,"context_line":"    def __str__(self):"},{"line_number":1967,"context_line":"        return \u0027\u003c%s, %s, %s, %s(%s), %s\u003e\u0027 \\"},{"line_number":1968,"context_line":"               % (self.timestamp, self.status, self._durable, self.shortfall,"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_b64374ae","line":1965,"in_reply_to":"9a1a9d01_bb15aa9c","updated":"2015-09-30 23:33:45.000000000","message":"Done","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":1993,"context_line":"            except (TypeError, ValueError):"},{"line_number":1994,"context_line":"                pass"},{"line_number":1995,"context_line":"        return {}"},{"line_number":1996,"context_line":""},{"line_number":1997,"context_line":"    def _get_bucket(self, timestamp):"},{"line_number":1998,"context_line":"        return self.buckets.setdefault("},{"line_number":1999,"context_line":"            timestamp, ECGetResponseBucket(self.policy.ec_ndata, timestamp))"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_6e567207","line":1996,"updated":"2015-09-30 12:56:04.000000000","message":"Could we re-use (or merge) _decode_fragments_pref in swift/obj/server.py? There is a small diffference the return value tho (dict or None). If we could place them to swift.common.utils it seems reasonable.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":1993,"context_line":"            except (TypeError, ValueError):"},{"line_number":1994,"context_line":"                pass"},{"line_number":1995,"context_line":"        return {}"},{"line_number":1996,"context_line":""},{"line_number":1997,"context_line":"    def _get_bucket(self, timestamp):"},{"line_number":1998,"context_line":"        return self.buckets.setdefault("},{"line_number":1999,"context_line":"            timestamp, ECGetResponseBucket(self.policy.ec_ndata, timestamp))"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_d02950e7","line":1996,"in_reply_to":"9a1a9d01_6e567207","updated":"2015-09-30 23:33:45.000000000","message":"nice! refactored to swift.common.utils.safe_json_loads","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":1996,"context_line":""},{"line_number":1997,"context_line":"    def _get_bucket(self, timestamp):"},{"line_number":1998,"context_line":"        return self.buckets.setdefault("},{"line_number":1999,"context_line":"            timestamp, ECGetResponseBucket(self.policy.ec_ndata, timestamp))"},{"line_number":2000,"context_line":""},{"line_number":2001,"context_line":"    def add_response(self, get, parts_iter):"},{"line_number":2002,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_4ecdae83","line":1999,"updated":"2015-09-30 12:56:04.000000000","message":"self.policy.ec_ndata is not quorum number. It should be  self.policy.quorum... wait, reading shortfall method definition, did we get wrong the argument (and variable) name of ECGetResponseBucket, right? That is because we need ec_ndate getters for decoding.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":1996,"context_line":""},{"line_number":1997,"context_line":"    def _get_bucket(self, timestamp):"},{"line_number":1998,"context_line":"        return self.buckets.setdefault("},{"line_number":1999,"context_line":"            timestamp, ECGetResponseBucket(self.policy.ec_ndata, timestamp))"},{"line_number":2000,"context_line":""},{"line_number":2001,"context_line":"    def add_response(self, get, parts_iter):"},{"line_number":2002,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_952e6aa3","line":1999,"in_reply_to":"9a1a9d01_4ecdae83","updated":"2015-09-30 23:33:45.000000000","message":"good point. I am using quorum to mean ec_ndata, but it has a different meaning elsewhere.\n\nwill fix","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2043,"context_line":"            # buckets with a quorum sort higher,"},{"line_number":2044,"context_line":"            # otherwise buckets with lowest shortfall_with_alts sort higher,"},{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_2c7c02b4","line":2046,"updated":"2015-09-30 12:56:04.000000000","message":"bucket.shortfall will be equal to or greater than 0 because max(result, 0 or 1) so that it should be bucket.shortfall \u003d\u003d 0, I guess it might be a nit, tho.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2043,"context_line":"            # buckets with a quorum sort higher,"},{"line_number":2044,"context_line":"            # otherwise buckets with lowest shortfall_with_alts sort higher,"},{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_70c2c4ea","line":2046,"in_reply_to":"9a1a9d01_2c7c02b4","updated":"2015-09-30 23:33:45.000000000","message":"you are correct but I prefer to keep the \u003c\u003d because it guards against someone removing the max() at some later time.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":2043,"context_line":"            # buckets with a quorum sort higher,"},{"line_number":2044,"context_line":"            # otherwise buckets with lowest shortfall_with_alts sort higher,"},{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_c42b48ea","line":2046,"in_reply_to":"9a1a9d01_70c2c4ea","updated":"2015-10-05 07:08:59.000000000","message":"Exactly, it seems okay to me.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_ece36ad7","line":2048,"updated":"2015-09-30 12:56:04.000000000","message":"if shortfall_with_alts has enough ec_ndata with durable timestamp, (-1 * bucket.shortfall_with_alts) will be 0? If it is true, this statement seems to be always False. Am I true? not sure.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_c47908fe","line":2048,"in_reply_to":"9a1a9d01_9633d0f5","updated":"2015-10-05 07:08:59.000000000","message":"OIC, \"boolean and int\" results in a integer and the reverse sort will order the buckets like that the bucket which has more available nodes prior to others. Seems reasonable. Thanks for your explanation!","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2045,"context_line":"            # finally buckets with newer timestamps sort higher."},{"line_number":2046,"context_line":"            quorum \u003d bucket.shortfall \u003c\u003d 0"},{"line_number":2047,"context_line":"            return (quorum,"},{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_9633d0f5","line":2048,"in_reply_to":"9a1a9d01_ece36ad7","updated":"2015-09-30 23:33:45.000000000","message":"if quorum\u003dTrue then this second statement is always False. So in this case the ordering is loft on shortfall_with_alts but I\u0027m not sure we care because we have a bucket with quorum\u003dTrue\n\nif quorum \u003d False then the second statement \u003d\u003d -1 * bucket.shortfall_with_alts\n\nbecause True and 0 -\u003e 0","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"},{"line_number":2052,"context_line":""},{"line_number":2053,"context_line":"    @property"},{"line_number":2054,"context_line":"    def best_bucket(self):"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_474fd136","line":2051,"updated":"2015-09-30 12:56:04.000000000","message":"Not sure but .values of dict will make a value list ordered by key in ascending order.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"},{"line_number":2052,"context_line":""},{"line_number":2053,"context_line":"    @property"},{"line_number":2054,"context_line":"    def best_bucket(self):"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_f6ad6ce9","line":2051,"in_reply_to":"9a1a9d01_474fd136","updated":"2015-09-30 23:33:45.000000000","message":"dict.values() does not guarantee any order AFAICT. But more importantly, here we are using a custom key function for the sort.\n\nApparently it is called a Decorate, Sort, Undecorate (DSU) pattern !","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":2048,"context_line":"                    not quorum and (-1 * bucket.shortfall_with_alts),"},{"line_number":2049,"context_line":"                    bucket.timestamp)"},{"line_number":2050,"context_line":""},{"line_number":2051,"context_line":"        return sorted(self.buckets.values(), key\u003dkey_fn, reverse\u003dTrue)"},{"line_number":2052,"context_line":""},{"line_number":2053,"context_line":"    @property"},{"line_number":2054,"context_line":"    def best_bucket(self):"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_af294d39","line":2051,"in_reply_to":"9a1a9d01_f6ad6ce9","updated":"2015-10-05 07:08:59.000000000","message":"Yeah, once I read this, I thought we can do more efficient sort because we have timestamp keys in the buckets dict. However, you set them into the last sorting key in the custom key. So that seems to work well.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03c7df4898b047111aebb4592f6422acc8f19d13","unresolved":false,"context_lines":[{"line_number":2011,"context_line":"        # The node may also have alternate fragments indexes (possibly at"},{"line_number":2012,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":2013,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":2014,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":2015,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027), {})"},{"line_number":2016,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":2017,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_88726310","line":2014,"updated":"2015-10-02 17:37:22.000000000","message":"This is how we \"recover\" from the optimistic GET receiving e.g.\n\n  ec_ndata-1 frags at t2 from nodes that also had t1.data+t1.durable\n  ec_nparity + 1 frags at t1\n\nIf we don\u0027t go back to alt nodes then a partial write at t2 prevents us getting to the last durable at t1, which is a regression from master.","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"358410d16db7bd464481ef3bdadd2fe88ff9e5a1","unresolved":false,"context_lines":[{"line_number":2071,"context_line":""},{"line_number":2072,"context_line":"    def get_extra_headers(self):"},{"line_number":2073,"context_line":"        frag_prefs \u003d self._get_frag_prefs()"},{"line_number":2074,"context_line":"        # TODO: change header name to X-Backend-Fragment-Preferences?"},{"line_number":2075,"context_line":"        return {\u0027X-Backend-Frag-Prefs\u0027: json.dumps(frag_prefs)}"},{"line_number":2076,"context_line":""},{"line_number":2077,"context_line":"    def _get_alternate_nodes(self):"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_c8b8772d","line":2074,"updated":"2015-10-02 06:09:39.000000000","message":"yes, I almost always prefer not to not use a an abbreviation","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8c2044b092e281e255e762578edd75d20fcbd60f","unresolved":false,"context_lines":[{"line_number":2071,"context_line":""},{"line_number":2072,"context_line":"    def get_extra_headers(self):"},{"line_number":2073,"context_line":"        frag_prefs \u003d self._get_frag_prefs()"},{"line_number":2074,"context_line":"        # TODO: change header name to X-Backend-Fragment-Preferences?"},{"line_number":2075,"context_line":"        return {\u0027X-Backend-Frag-Prefs\u0027: json.dumps(frag_prefs)}"},{"line_number":2076,"context_line":""},{"line_number":2077,"context_line":"    def _get_alternate_nodes(self):"}],"source_content_type":"text/x-python","patch_set":20,"id":"3a29b11f_1583aa99","line":2074,"in_reply_to":"9a1a9d01_c8b8772d","updated":"2015-10-20 14:23:39.000000000","message":"Done","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"03c7df4898b047111aebb4592f6422acc8f19d13","unresolved":false,"context_lines":[{"line_number":2071,"context_line":""},{"line_number":2072,"context_line":"    def get_extra_headers(self):"},{"line_number":2073,"context_line":"        frag_prefs \u003d self._get_frag_prefs()"},{"line_number":2074,"context_line":"        # TODO: change header name to X-Backend-Fragment-Preferences?"},{"line_number":2075,"context_line":"        return {\u0027X-Backend-Frag-Prefs\u0027: json.dumps(frag_prefs)}"},{"line_number":2076,"context_line":""},{"line_number":2077,"context_line":"    def _get_alternate_nodes(self):"}],"source_content_type":"text/x-python","patch_set":20,"id":"9a1a9d01_c86dbb9a","line":2074,"in_reply_to":"9a1a9d01_c8b8772d","updated":"2015-10-02 17:37:22.000000000","message":"OK. I had some feedback from tdasliva to the effect that \u0027preferences\u0027 was confusing w.r.t. it being an exclusion list rather than inclusion.\n\nOther ideas for the name:\n\nX-Backend-If-Frag-Not\nX-Backend-Fragment-Hints\n\nbut I\u0027d be happy with X-Backend-Fragment-Preferences","commit_id":"ae5ce4048e303de3618bc94ab1fe0abe660b6849"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":1921,"context_line":"    A helper class to encapsulate the properties of buckets in which fragment"},{"line_number":1922,"context_line":"    getters and alternate nodes are collected."},{"line_number":1923,"context_line":"    \"\"\""},{"line_number":1924,"context_line":"    def __init__(self, policy, timestamp):"},{"line_number":1925,"context_line":"        self.policy \u003d policy"},{"line_number":1926,"context_line":"        self.timestamp \u003d timestamp"},{"line_number":1927,"context_line":"        self.gets \u003d collections.defaultdict(list)"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_b16132d5","line":1924,"updated":"2016-02-08 10:29:54.000000000","message":"I\u0027d like to add a note the *timestamp* is actually timestamp str which equals to the value \"X-Timestamp\" to prevent the misunderstanding like as Timestamp instance.\n(Or I like to use Timestamp instance instead but it seems to cause an error, see my comment at L2092.)","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":1921,"context_line":"    A helper class to encapsulate the properties of buckets in which fragment"},{"line_number":1922,"context_line":"    getters and alternate nodes are collected."},{"line_number":1923,"context_line":"    \"\"\""},{"line_number":1924,"context_line":"    def __init__(self, policy, timestamp):"},{"line_number":1925,"context_line":"        self.policy \u003d policy"},{"line_number":1926,"context_line":"        self.timestamp \u003d timestamp"},{"line_number":1927,"context_line":"        self.gets \u003d collections.defaultdict(list)"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_ff8d43d3","line":1924,"in_reply_to":"5a5ae5dd_b16132d5","updated":"2016-02-12 18:21:33.000000000","message":"Done - good call.\n\nWe could go through code converting everything to Timestamps but I\u0027m not sure its worth it in this context when we don\u0027t need any of the methods a Timestamp offers?","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2089,"context_line":"    def get_extra_headers(self):"},{"line_number":2090,"context_line":"        frag_prefs \u003d self._get_frag_prefs()"},{"line_number":2091,"context_line":"        # TODO: change header name to X-Backend-Fragment-Preferences?"},{"line_number":2092,"context_line":"        return {\u0027X-Backend-Fragment-Preferences\u0027: json.dumps(frag_prefs)}"},{"line_number":2093,"context_line":""},{"line_number":2094,"context_line":"    def _get_alternate_nodes(self):"},{"line_number":2095,"context_line":"        if self.node_iter_count \u003c\u003d self.policy.ec_ndata:"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_91951634","line":2092,"updated":"2016-02-08 10:29:54.000000000","message":"For now, this works well because frag_prefs consists of a list with {\u0027timestamp\u0027: Timestamp(ts).internal, \u0027exclude\u0027: \u003ca list of int(fragment_index)\u003e}. That\u0027s safe.\n\nHowever, json.dumps({Timestamp(ts): \u0027something\u0027}) (i.e. direct encoding from Timestamp instance) seems unsafe operation when I tested in local environment, I didn\u0027t get the reason (we have __repr__ method for serialization in Timestamp itself) though.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":2238,"context_line":"            for _junk in range(policy.ec_ndata):"},{"line_number":2239,"context_line":"                pile.spawn(self._fragment_GET_request,"},{"line_number":2240,"context_line":"                           req, safe_iter, partition,"},{"line_number":2241,"context_line":"                           policy, buckets.get_extra_headers)"},{"line_number":2242,"context_line":""},{"line_number":2243,"context_line":"            bad_bucket \u003d ECGetResponseBucket(policy, None)"},{"line_number":2244,"context_line":"            bad_bucket.set_durable()"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_00530a6c","line":2241,"updated":"2016-02-08 10:29:54.000000000","message":"Can we get rid of adding get_extra_headers to header_provider for the first GET requests (i think at this time, probably frag pref is empty)\n\nI mean the semantic difference between \"frag_prefs specified but empty\" vs \"no frag_prefs specified\" at Object-Server.\n\nI think, when it\u0027s not specified, that means \"hey, any frags/timestamps are ok to get back the successful response\". However if specified, that means \"strictly we don\u0027t allow to get successful code if the specified object is not available\"\n\nThat is just my opinion, how do you think for this?\n\n\nAnyways, I got some failures from test/unit/proxy/controllers/test_obj.py when I tried to remove get_extra_headers temprorary :/","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":2238,"context_line":"            for _junk in range(policy.ec_ndata):"},{"line_number":2239,"context_line":"                pile.spawn(self._fragment_GET_request,"},{"line_number":2240,"context_line":"                           req, safe_iter, partition,"},{"line_number":2241,"context_line":"                           policy, buckets.get_extra_headers)"},{"line_number":2242,"context_line":""},{"line_number":2243,"context_line":"            bad_bucket \u003d ECGetResponseBucket(policy, None)"},{"line_number":2244,"context_line":"            bad_bucket.set_durable()"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_e26b6a83","line":2241,"in_reply_to":"5a5ae5dd_00530a6c","updated":"2016-02-12 18:21:33.000000000","message":"X-Backend-Fragment-Preferences \u003d [] (empty list) means \"get any frag at any timestamp, *even if not durable*\".\n\nNot setting X-Backend-Fragment-Preferences means \"get any frag at any timestamp that IS durable\"\n\nSo IIRC it is a deliberate choice to call get_extra_headers for the first GETs and then the bucket controller selects bucket which did get a durable timestamp frag.\n\nAs soon as we receive any frag responses, the preferences list is populated but only with frag indexes/timestamps that the proxy has received a response for - so we only ever exclude frags that we have already found.\n\nNote: we have discussed removing the requirement for any durable fragment altogether, but this patch does not go that far.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1859,"context_line":"        self.status \u003d getter.last_status"},{"line_number":1860,"context_line":"        headers \u003d getter.last_headers"},{"line_number":1861,"context_line":"        # TODO: we could select self.headers from source with latest"},{"line_number":1862,"context_line":"        # X-Timestamp in case some sources have a newer .meta file than others."},{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_e5ff45a8","line":1862,"updated":"2016-09-07 12:04:21.000000000","message":"Is this why you don\u0027t assign getter.last_headers to self.headers directly?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":1859,"context_line":"        self.status \u003d getter.last_status"},{"line_number":1860,"context_line":"        headers \u003d getter.last_headers"},{"line_number":1861,"context_line":"        # TODO: we could select self.headers from source with latest"},{"line_number":1862,"context_line":"        # X-Timestamp in case some sources have a newer .meta file than others."},{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_95926856","line":1862,"in_reply_to":"9a89bdaa_432c3699","updated":"2016-09-14 10:28:27.000000000","message":"Sweet. That sounds like optimistic GET metadata and agree we  can improve that behavior in another patch. Looks like the new inline docs for TODO is awesome to know what we should do in the follow up. Done.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1859,"context_line":"        self.status \u003d getter.last_status"},{"line_number":1860,"context_line":"        headers \u003d getter.last_headers"},{"line_number":1861,"context_line":"        # TODO: we could select self.headers from source with latest"},{"line_number":1862,"context_line":"        # X-Timestamp in case some sources have a newer .meta file than others."},{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_432c3699","line":1862,"in_reply_to":"9a89bdaa_e5ff45a8","updated":"2016-09-13 18:49:29.000000000","message":"no, that is just my redundant code ! BTW I think the TODO can be a follow up - a replicated GET may return older metadata than the newest on disk, since we just randomly choose one replica which may not have the latest .meta file. So this current behavior is not worse.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1862,"context_line":"        # X-Timestamp in case some sources have a newer .meta file than others."},{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"},{"line_number":1866,"context_line":"        self.gets[frag_index].append((getter, parts_iter))"},{"line_number":1867,"context_line":""},{"line_number":1868,"context_line":"    def get_responses(self):"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_45ac71a0","line":1865,"range":{"start_line":1865,"start_character":36,"end_line":1865,"end_character":78},"updated":"2016-09-07 12:04:21.000000000","message":"nits) Either is fine though, \"if frag_index is not None else None\" seems clear what assigned to frag_index.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1862,"context_line":"        # X-Timestamp in case some sources have a newer .meta file than others."},{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"},{"line_number":1866,"context_line":"        self.gets[frag_index].append((getter, parts_iter))"},{"line_number":1867,"context_line":""},{"line_number":1868,"context_line":"    def get_responses(self):"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_c7721be3","line":1865,"range":{"start_line":1865,"start_character":36,"end_line":1865,"end_character":78},"in_reply_to":"9a89bdaa_45ac71a0","updated":"2016-09-13 18:49:29.000000000","message":"Done. yes, that\u0027s much better!","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"},{"line_number":1866,"context_line":"        self.gets[frag_index].append((getter, parts_iter))"},{"line_number":1867,"context_line":""},{"line_number":1868,"context_line":"    def get_responses(self):"},{"line_number":1869,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_257dade7","line":1866,"updated":"2016-09-07 12:04:21.000000000","message":"memo) I thought why we can get the None case here but it seems for a bad bucket. ok, make sense.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1863,"context_line":"        self.headers \u003d headers"},{"line_number":1864,"context_line":"        frag_index \u003d headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":1865,"context_line":"        frag_index \u003d int(frag_index) if frag_index is not None else frag_index"},{"line_number":1866,"context_line":"        self.gets[frag_index].append((getter, parts_iter))"},{"line_number":1867,"context_line":""},{"line_number":1868,"context_line":"    def get_responses(self):"},{"line_number":1869,"context_line":"        \"\"\""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_032f8e88","line":1866,"in_reply_to":"9a89bdaa_257dade7","updated":"2016-09-13 18:49:29.000000000","message":"yes, bad bucket \u003d\u003d index None","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1952,"context_line":"        # maintain multiple buckets keyed by (timestamp, etag). Ditto for"},{"line_number":1953,"context_line":"        # status code."},{"line_number":1954,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1955,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1956,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"        # The node may also have alternate fragments indexes (possibly at"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_ef09df2d","line":1955,"updated":"2016-09-07 12:04:21.000000000","message":"This means we want to handle them as different if object-server returns only X-Timestamp, right? That is because X-Backend-Data-Timestamp and X-Backend-Timestamp is Timestamp.internal but X-Timestamp is Timestamp.normal and response bucket handle it as just a string.\n\nIt seems safe because object-server should return X-Backend-Timestamp except particular error cases, though.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":1952,"context_line":"        # maintain multiple buckets keyed by (timestamp, etag). Ditto for"},{"line_number":1953,"context_line":"        # status code."},{"line_number":1954,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1955,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1956,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"        # The node may also have alternate fragments indexes (possibly at"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_55170051","line":1955,"in_reply_to":"9a89bdaa_3ea105f9","updated":"2016-09-14 10:28:27.000000000","message":"Considering the EC specification, the bucket should gather the same *Timestamp.internal* fragments, i think. That is because the offset will be incremented by (basically) reconciler, AFAIK. Hence, those objects which have same timestamp but different offset are written to different placements/policies in the split brain case, that seems to be different object each other.\n\nSo current code is still safe, we should prior Timestamp.internal to Timestamp.normal anyway, IMHO.\n\nJust wondering why/what is saved for the L1955 though. At first, I thought it saves some error cases like 507 or 404 (w/o tombstone) but it doesn\u0027t enter to the EcGetResponseCollection but to EcGetResponseBucket.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1952,"context_line":"        # maintain multiple buckets keyed by (timestamp, etag). Ditto for"},{"line_number":1953,"context_line":"        # status code."},{"line_number":1954,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1955,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1956,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1957,"context_line":""},{"line_number":1958,"context_line":"        # The node may also have alternate fragments indexes (possibly at"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_3ea105f9","line":1955,"in_reply_to":"9a89bdaa_ef09df2d","updated":"2016-09-13 18:49:29.000000000","message":"I don\u0027t recall exactly why I added the default to X-Timestamp. Perhaps because a really old object server versions would not return X-Backend-Timestamp?? I think you make a good point that if we only have X-Timestamp then we should at least convert the string format to be the full internal format.\n\nHmmm, now I am wondering if we can expect to ever (in some edge case) have frags of the same object with different timestamp *offsets* so that their internal formats are different :/ Maybe I should be converting to the normal format i.e. the client-facing timestamp??","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1959,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":1960,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":1961,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":1962,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027), {})"},{"line_number":1963,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":1964,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"},{"line_number":1965,"context_line":"        # If the response includes a durable timestamp then mark that bucket as"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_69786ff3","line":1962,"updated":"2016-09-07 12:04:21.000000000","message":"Looking at the usage of \"safe_json_loads\" across modules, I noticed the difference of json format with \"X-Backend-Fragments\" and \"X-Backend-Fragment-Preferences\", though both headers wants to make a json mapping timestamp and indexes.\n\nFrom your tests in this patch, X-Backend-Fragments is a list of dict like,\n\n backend_frags \u003d json.dumps({ts_0.internal: [0],\n                             ts_2.internal: [2]})\n\nHowever, X-Backend-Fragment-Preferences is dict of lists like,\n\n prefs \u003d [{\u0027timestamp\u0027: ts_1.internal, \u0027exclude\u0027: [3]},\n          {\u0027timestamp\u0027: ts_2.internal, \u0027exclude\u0027: []},\n          {\u0027timestamp\u0027: ts_3.internal, \u0027exclude\u0027: []}]\n\nI think that\u0027s why we need \"default\" kwargs for safe_json_loads. If we could use same format, list of dicts or dict of lists, could we avoid to set the default value {}, here?\n\nI\u0027m not sure on this, just wondering right now, though.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1959,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":1960,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":1961,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":1962,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027), {})"},{"line_number":1963,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":1964,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"},{"line_number":1965,"context_line":"        # If the response includes a durable timestamp then mark that bucket as"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_83b0d044","line":1962,"in_reply_to":"9a89bdaa_69786ff3","updated":"2016-09-13 18:49:29.000000000","message":"I would rather not change the format of the header values: X-Backend-Fragment-Preferences is a list of dicts because there is significance in the order of the list. X-Backend-Fragments does not need to be so verbose, a dict is sufficient.\n\nBut I think I can simplify and avoid the \u0027default\u0027 param by doing this:\n\n  frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027)) or {}","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":1959,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":1960,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":1961,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":1962,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027), {})"},{"line_number":1963,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":1964,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"},{"line_number":1965,"context_line":"        # If the response includes a durable timestamp then mark that bucket as"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_e025a8ae","line":1962,"in_reply_to":"9a89bdaa_83b0d044","updated":"2016-09-14 10:28:27.000000000","message":"Ok, and nice improvement. Thanks.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2174,"context_line":"            for _junk in range(policy.ec_ndata):"},{"line_number":2175,"context_line":"                pile.spawn(self._fragment_GET_request,"},{"line_number":2176,"context_line":"                           req, safe_iter, partition,"},{"line_number":2177,"context_line":"                           policy, buckets.get_extra_headers)"},{"line_number":2178,"context_line":""},{"line_number":2179,"context_line":"            bad_bucket \u003d ECGetResponseBucket(policy, None)"},{"line_number":2180,"context_line":"            bad_bucket.set_durable()"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_8fc0532c","line":2177,"range":{"start_line":2177,"start_character":35,"end_line":2177,"end_character":60},"updated":"2016-09-07 12:04:21.000000000","message":"memo) I thought, at the first time to gather fragments, we don\u0027t need any extra prefs because empty ECGetResponseCollection set nothing but it\u0027s wrong.\n\nThe empty X-Backend-Fragment-Preferences forces object-server to report available fragments insead of 404 if it exists even if it\u0027s not durable. That makes  me sense. Is that described somewhere? I\u0027m not sure if I may be missing the docs.\n\nThat is why we have to serve get_extra_headers for the first _fragment_GET_request.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2174,"context_line":"            for _junk in range(policy.ec_ndata):"},{"line_number":2175,"context_line":"                pile.spawn(self._fragment_GET_request,"},{"line_number":2176,"context_line":"                           req, safe_iter, partition,"},{"line_number":2177,"context_line":"                           policy, buckets.get_extra_headers)"},{"line_number":2178,"context_line":""},{"line_number":2179,"context_line":"            bad_bucket \u003d ECGetResponseBucket(policy, None)"},{"line_number":2180,"context_line":"            bad_bucket.set_durable()"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_8396104a","line":2177,"range":{"start_line":2177,"start_character":35,"end_line":2177,"end_character":60},"in_reply_to":"9a89bdaa_8fc0532c","updated":"2016-09-13 18:49:29.000000000","message":"Correct, sending empty X-Backend-Fragment-Preferences triggers the object server to return non-durable frags. The object server does not do that by default because it does not know that the proxy has been upgraded to this version. \n\nI\u0027ll add a comment.\n\nAlso note that because these _fragment_GET_request call execute in threads, it is possible for the bucket collection to have had a response added before a slower initial _fragment_GET_request threads calls buckets.get_extra_headers. Probably unlikely, but possible.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2195,"context_line":"                    # which definitely happens in tests and results in status"},{"line_number":2196,"context_line":"                    # of None. We should fix that but keep this guard because"},{"line_number":2197,"context_line":"                    # there is also a race between testing nodes_left/spawning"},{"line_number":2198,"context_line":"                    # a getter and an existing getter calling next(node_iter)."},{"line_number":2199,"context_line":"                    continue"},{"line_number":2200,"context_line":"                if is_success(get.last_status):"},{"line_number":2201,"context_line":"                    # all 2xx responses are managed by a response collection"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_35ee76b4","line":2198,"updated":"2016-09-07 12:04:21.000000000","message":"Yeah, that could happen around https://github.com/openstack/swift/blob/master/swift/proxy/controllers/base.py#L1117-L1121\n\nanyway, if last_status is None in the getter, we can skip it as unavailable node.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2195,"context_line":"                    # which definitely happens in tests and results in status"},{"line_number":2196,"context_line":"                    # of None. We should fix that but keep this guard because"},{"line_number":2197,"context_line":"                    # there is also a race between testing nodes_left/spawning"},{"line_number":2198,"context_line":"                    # a getter and an existing getter calling next(node_iter)."},{"line_number":2199,"context_line":"                    continue"},{"line_number":2200,"context_line":"                if is_success(get.last_status):"},{"line_number":2201,"context_line":"                    # all 2xx responses are managed by a response collection"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_6684ba0a","line":2198,"in_reply_to":"9a89bdaa_35ee76b4","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2233,"context_line":"            app_iter \u003d ECAppIter("},{"line_number":2234,"context_line":"                req.swift_entity_path,"},{"line_number":2235,"context_line":"                policy,"},{"line_number":2236,"context_line":"                [iter for _getter, iter in best_bucket.get_responses()],"},{"line_number":2237,"context_line":"                range_specs, fa_length, obj_length,"},{"line_number":2238,"context_line":"                self.app.logger)"},{"line_number":2239,"context_line":"            resp \u003d Response("}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_0d05e644","line":2236,"range":{"start_line":2236,"start_character":16,"end_line":2236,"end_character":38},"updated":"2016-09-07 12:04:21.000000000","message":"Avoid using \"iter\" which is reserved for built-in function.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2233,"context_line":"            app_iter \u003d ECAppIter("},{"line_number":2234,"context_line":"                req.swift_entity_path,"},{"line_number":2235,"context_line":"                policy,"},{"line_number":2236,"context_line":"                [iter for _getter, iter in best_bucket.get_responses()],"},{"line_number":2237,"context_line":"                range_specs, fa_length, obj_length,"},{"line_number":2238,"context_line":"                self.app.logger)"},{"line_number":2239,"context_line":"            resp \u003d Response("}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_517e8e6a","line":2236,"range":{"start_line":2236,"start_character":16,"end_line":2236,"end_character":38},"in_reply_to":"9a89bdaa_0d05e644","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":2259,"context_line":"            reasons \u003d []"},{"line_number":2260,"context_line":"            bodies \u003d []"},{"line_number":2261,"context_line":"            headers \u003d []"},{"line_number":2262,"context_line":"            for getter, _iter in bad_bucket.get_responses():"},{"line_number":2263,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2264,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2265,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_006bedb7","line":2262,"updated":"2016-09-07 12:04:21.000000000","message":"Should we close all iterators if we don\u0027t use anymore as well as ECAppIter[1]?\n\nI\u0027m not sure if it\u0027s necessary. And also we may need to close all iterators for successful buckets without enough ec_n responses.\n\n1: https://github.com/openstack/swift/blob/master/swift/proxy/controllers/obj.py#L940-L941","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2259,"context_line":"            reasons \u003d []"},{"line_number":2260,"context_line":"            bodies \u003d []"},{"line_number":2261,"context_line":"            headers \u003d []"},{"line_number":2262,"context_line":"            for getter, _iter in bad_bucket.get_responses():"},{"line_number":2263,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2264,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2265,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_ec2c294c","line":2262,"in_reply_to":"9a89bdaa_006bedb7","updated":"2016-09-13 18:49:29.000000000","message":"Not sure. I don\u0027t see any of the unused part_iters being closed on master. We never call the kickoff method on them, but I\u0027m not sure if that makes a difference.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":2259,"context_line":"            reasons \u003d []"},{"line_number":2260,"context_line":"            bodies \u003d []"},{"line_number":2261,"context_line":"            headers \u003d []"},{"line_number":2262,"context_line":"            for getter, _iter in bad_bucket.get_responses():"},{"line_number":2263,"context_line":"                statuses.extend(getter.statuses)"},{"line_number":2264,"context_line":"                reasons.extend(getter.reasons)"},{"line_number":2265,"context_line":"                bodies.extend(getter.bodies)"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_80f6a4d5","line":2262,"in_reply_to":"9a89bdaa_ec2c294c","updated":"2016-09-14 10:28:27.000000000","message":"Fair point, I confirmed this is match with current swift/proxy/controllers/obj.py [1].\n\nOkay, fix later if the problem exists. This patch doesn\u0027t change any behaviors at this time.\n\n\n1: https://github.com/openstack/swift/blob/master/swift/proxy/controllers/obj.py#L1996","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":1960,"context_line":"        # status code."},{"line_number":1961,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1962,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1963,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1964,"context_line":""},{"line_number":1965,"context_line":"        # The node may also have alternate fragments indexes (possibly at"},{"line_number":1966,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_773cee93","line":1963,"updated":"2016-09-14 10:37:45.000000000","message":"Ah, wait... Are we missing etag check which exists in the current master[1]?\n\nWhy...test_mixing_different_objects_fragment_archives doesn\u0027t fail...ah, gotcha. The reason why it doesn\u0027t fail is it\u0027s using different timestamp so that they are in different buckets. However, I\u0027m still wondering we don\u0027t never get same timestamp but different content (but same length)? I think the answer is \"no it could happen in the eventual consistency world\". And in that situation, current code can feed garbage (fragments with different etags) to pyeclib to server wrong content with no errors. Maybe? Could you double check?\n\n1: https://github.com/openstack/swift/blob/master/swift/proxy/controllers/obj.py#L1945\n\n2: https://github.com/openstack/swift/blob/master/test/unit/proxy/test_server.py#L5516","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd672b6f8a8f2bd7f79dee08b81fc673ae55f1aa","unresolved":false,"context_lines":[{"line_number":1960,"context_line":"        # status code."},{"line_number":1961,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1962,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1963,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1964,"context_line":""},{"line_number":1965,"context_line":"        # The node may also have alternate fragments indexes (possibly at"},{"line_number":1966,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_943cbcc9","line":1963,"in_reply_to":"7a8ec9b2_773cee93","updated":"2016-09-15 18:11:07.000000000","message":"I had a TODO at line 1958 on this topic...so, I do not think that frags for objects with different content but at same timestamp is an expected case - the whole consistency mechanism of swift relies on timestamp being the measure of object consistency e.g. reconstructor would not detect any inconsistency if we had frags at same time with different etags because the suffix hashing is based on timestamps.\n\nGiven that, I have not attempted to manage a bucket for each (timestamp, etag) tuple (believe me - it is *really* hard to make the optimistic gets heuristics work if you go down that route because you cannot know which etag any alternate frags may have). But I will add a check and ignore any frags with mismatching etags simply so that we do not cause pyeclib to blow up IF this ever happened.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e67a322b4ab5a46bfcbffee39e85e21716086328","unresolved":false,"context_lines":[{"line_number":1960,"context_line":"        # status code."},{"line_number":1961,"context_line":"        t_data_file \u003d headers.get(\u0027X-Backend-Data-Timestamp\u0027)"},{"line_number":1962,"context_line":"        t_obj \u003d headers.get(\u0027X-Backend-Timestamp\u0027, headers.get(\u0027X-Timestamp\u0027))"},{"line_number":1963,"context_line":"        self._get_bucket(t_data_file or t_obj).add_response(get, parts_iter)"},{"line_number":1964,"context_line":""},{"line_number":1965,"context_line":"        # The node may also have alternate fragments indexes (possibly at"},{"line_number":1966,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_af0bfad4","line":1963,"in_reply_to":"7a8ec9b2_943cbcc9","updated":"2016-09-16 11:25:18.000000000","message":"This one is addressed in the newest patch set, done.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":1966,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":1967,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":1968,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":1969,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027)) or {}"},{"line_number":1970,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":1971,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"},{"line_number":1972,"context_line":"        # If the response includes a durable timestamp then mark that bucket as"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_002b7483","line":1969,"range":{"start_line":1969,"start_character":72,"end_line":1969,"end_character":77},"updated":"2016-09-14 10:37:45.000000000","message":"Love this!","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd672b6f8a8f2bd7f79dee08b81fc673ae55f1aa","unresolved":false,"context_lines":[{"line_number":1966,"context_line":"        # different timestamps). For each list of alternate fragments indexes,"},{"line_number":1967,"context_line":"        # find the bucket for their data file timestamp and add the node and"},{"line_number":1968,"context_line":"        # list to that bucket\u0027s alternate nodes."},{"line_number":1969,"context_line":"        frag_sets \u003d safe_json_loads(headers.get(\u0027X-Backend-Fragments\u0027)) or {}"},{"line_number":1970,"context_line":"        for t_frag, frag_set in frag_sets.items():"},{"line_number":1971,"context_line":"            self._get_bucket(t_frag).add_alternate_nodes(get.node, frag_set)"},{"line_number":1972,"context_line":"        # If the response includes a durable timestamp then mark that bucket as"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_d432c4f1","line":1969,"range":{"start_line":1969,"start_character":72,"end_line":1969,"end_character":77},"in_reply_to":"7a8ec9b2_002b7483","updated":"2016-09-15 18:11:07.000000000","message":"Done","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":2191,"context_line":"            best_bucket \u003d None"},{"line_number":2192,"context_line":"            extra_requests \u003d 0"},{"line_number":2193,"context_line":"            # max_extra_requests is an arbitrary hard limit for spawning extra"},{"line_number":2194,"context_line":"            # getters in case some unforeseen scenario, or a misbehaving object"},{"line_number":2195,"context_line":"            # server, causes us to otherwise make endless requests e.g. if an"},{"line_number":2196,"context_line":"            # object server were to ignore frag_prefs and always respond with"},{"line_number":2197,"context_line":"            # a frag that is already in a bucket."}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_9b00e4ac","line":2194,"range":{"start_line":2194,"start_character":61,"end_line":2194,"end_character":72},"updated":"2016-09-14 03:15:04.000000000","message":"Or old! As a result, we don\u0027t care about the order in which object and proxy nodes upgrade, right?","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":2191,"context_line":"            best_bucket \u003d None"},{"line_number":2192,"context_line":"            extra_requests \u003d 0"},{"line_number":2193,"context_line":"            # max_extra_requests is an arbitrary hard limit for spawning extra"},{"line_number":2194,"context_line":"            # getters in case some unforeseen scenario, or a misbehaving object"},{"line_number":2195,"context_line":"            # server, causes us to otherwise make endless requests e.g. if an"},{"line_number":2196,"context_line":"            # object server were to ignore frag_prefs and always respond with"},{"line_number":2197,"context_line":"            # a frag that is already in a bucket."}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_8bd93586","line":2194,"range":{"start_line":2194,"start_character":61,"end_line":2194,"end_character":72},"in_reply_to":"7a8ec9b2_9b00e4ac","updated":"2016-09-14 17:32:23.000000000","message":"we shouldn\u0027t need to care about upgrade order (but I should go and run some functional tests with various server versions). An older object server will never inform the proxy of alternate fragments via x-backend-fragments, so that node will neot make it into a bucket\u0027s alternate nodes and the proxy should never issue an extra request to it.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":15343,"name":"Tim Burke","email":"tburke@nvidia.com","username":"tburke"},"change_message_id":"d6565158bc3cab9c6223be05a0be44fe84dcace2","unresolved":false,"context_lines":[{"line_number":2252,"context_line":"                headers\u003dresp_headers,"},{"line_number":2253,"context_line":"                conditional_response\u003dTrue,"},{"line_number":2254,"context_line":"                app_iter\u003dapp_iter)"},{"line_number":2255,"context_line":"            resp.accept_ranges \u003d \u0027bytes\u0027"},{"line_number":2256,"context_line":"            try:"},{"line_number":2257,"context_line":"                app_iter.kickoff(req, resp)"},{"line_number":2258,"context_line":"            except HTTPException as err_resp:"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_5bcbbc72","line":2255,"updated":"2016-09-14 03:15:04.000000000","message":"Is this necessary? I thought this was taken care of in _fix_response.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"7026af1c1a945eaa4f0f53feb6ca5c0605c43b02","unresolved":false,"context_lines":[{"line_number":2252,"context_line":"                headers\u003dresp_headers,"},{"line_number":2253,"context_line":"                conditional_response\u003dTrue,"},{"line_number":2254,"context_line":"                app_iter\u003dapp_iter)"},{"line_number":2255,"context_line":"            resp.accept_ranges \u003d \u0027bytes\u0027"},{"line_number":2256,"context_line":"            try:"},{"line_number":2257,"context_line":"                app_iter.kickoff(req, resp)"},{"line_number":2258,"context_line":"            except HTTPException as err_resp:"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_cbedfdf0","line":2255,"in_reply_to":"7a8ec9b2_5bcbbc72","updated":"2016-09-14 17:32:23.000000000","message":"Agree, probably crept through in a rebase. will remove.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1451,"context_line":"                     # data newer than durable ignored, even if its only data"},{"line_number":1452,"context_line":"                     [(\u00270000000008.00000#1.data\u0027, False, True),"},{"line_number":1453,"context_line":"                      (\u00270000000007.00000.durable\u0027, False, False)],"},{"line_number":1454,"context_line":""},{"line_number":1455,"context_line":"                     # data older than durable is ignored"},{"line_number":1456,"context_line":"                     [(\u00270000000007.00000.durable\u0027, \u0027.durable\u0027),"},{"line_number":1457,"context_line":"                      (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027),"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_409ed563","side":"PARENT","line":1454,"updated":"2016-09-07 12:04:21.000000000","message":"Ok, those test have been moved to L1487 as the case w/o frag_prefs for backward compatibility.","commit_id":"dd30b9ef984f3746a8d09bfcff751825921aab75"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":605,"context_line":"                ext[1:] + \u0027_file\u0027: os.path.join("},{"line_number":606,"context_line":"                    class_under_test._datadir, filename)"},{"line_number":607,"context_line":"                for (filename, ext) in [v[:2] for v in test]"},{"line_number":608,"context_line":"                if ext in (\u0027.data\u0027, \u0027.meta\u0027, \u0027.ts\u0027)}"},{"line_number":609,"context_line":"            # list(zip(...)) for py3 compatibility (zip is lazy there)"},{"line_number":610,"context_line":"            files \u003d list(list(zip(*test))[0])"},{"line_number":611,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_94673b4c","line":608,"range":{"start_line":608,"start_character":16,"end_line":608,"end_character":51},"updated":"2016-09-07 12:04:21.000000000","message":"Probably we need to add \u0027.durable\u0027 here as an ext. Otherwise, all scenarios which want to assert if \u0027.durable\u0027 info exists there doesn\u0027t nothing. Please see my comment at L1623.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":605,"context_line":"                ext[1:] + \u0027_file\u0027: os.path.join("},{"line_number":606,"context_line":"                    class_under_test._datadir, filename)"},{"line_number":607,"context_line":"                for (filename, ext) in [v[:2] for v in test]"},{"line_number":608,"context_line":"                if ext in (\u0027.data\u0027, \u0027.meta\u0027, \u0027.ts\u0027)}"},{"line_number":609,"context_line":"            # list(zip(...)) for py3 compatibility (zip is lazy there)"},{"line_number":610,"context_line":"            files \u003d list(list(zip(*test))[0])"},{"line_number":611,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_73edf18a","line":608,"range":{"start_line":608,"start_character":16,"end_line":608,"end_character":51},"in_reply_to":"9a89bdaa_4a016b04","updated":"2016-09-14 10:28:27.000000000","message":"Oh, ok. I was wrong. This test is for check the return value for [1].\n\nHmmm, I\u0027m wondering if we could make assertions for something for durable (mabye durable_frag_set?). Since this commit, those tests cannot assert if that is durable or not. With older version, we can assert it because diskfile reported only durable file, right? I wrote a small piece of my thought on my gist[2]. I think it\u0027s worth we can assert the actual status of files including a durable file. Thought?\n\n1: https://github.com/openstack/swift/blob/master/swift/obj/diskfile.py#L748-L751\n2: https://gist.github.com/bloodeagle40234/f4b5b0aa430384762f13b10d016d6c6a","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":605,"context_line":"                ext[1:] + \u0027_file\u0027: os.path.join("},{"line_number":606,"context_line":"                    class_under_test._datadir, filename)"},{"line_number":607,"context_line":"                for (filename, ext) in [v[:2] for v in test]"},{"line_number":608,"context_line":"                if ext in (\u0027.data\u0027, \u0027.meta\u0027, \u0027.ts\u0027)}"},{"line_number":609,"context_line":"            # list(zip(...)) for py3 compatibility (zip is lazy there)"},{"line_number":610,"context_line":"            files \u003d list(list(zip(*test))[0])"},{"line_number":611,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_4a016b04","line":608,"range":{"start_line":608,"start_character":16,"end_line":608,"end_character":51},"in_reply_to":"9a89bdaa_94673b4c","updated":"2016-09-13 18:49:29.000000000","message":"See reply at L1623...no, there is never a durable_file in the _get_ondisk_files results dict\n\nThe second item in the test tuple scenarios is also used with test_cleanup_ondisk_files and is needed for durable files in that test.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1620,"context_line":"            [(\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1621,"context_line":"             (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027, True),"},{"line_number":1622,"context_line":"             (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1623,"context_line":"             (\u00270000000006.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_b497df38","line":1623,"updated":"2016-09-07 12:04:21.000000000","message":"This looks odd because if the newer .data file exists, diskfile doesn\u0027t report the .durable file here. I found the reason why this still works and make a comment at L608.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":1620,"context_line":"            [(\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1621,"context_line":"             (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027, True),"},{"line_number":1622,"context_line":"             (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1623,"context_line":"             (\u00270000000006.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_6bc16de4","line":1623,"in_reply_to":"9a89bdaa_47bc06dc","updated":"2016-09-14 10:28:27.000000000","message":"Yes, I\u0027d love to reuse that flag for assertions of durable_frag_set in the result dict.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1620,"context_line":"            [(\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1621,"context_line":"             (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027, True),"},{"line_number":1622,"context_line":"             (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1623,"context_line":"             (\u00270000000006.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_47bc06dc","line":1623,"in_reply_to":"9a89bdaa_b497df38","updated":"2016-09-13 18:49:29.000000000","message":"This does look odd because _test_get_ondisk_files does not check for a durable_file in the results of _get_ondisk_files (that key is never expected). The inclusion of the \u0027.durable\u0027 in the tuple is for when the same scenarios are passed to _test_cleanup_ondisk_files_files which does not happen here, but could happen - I will fix that.\n\nBTW, the second item of tuple does not need to be \u0027.durable\u0027, it just needs to be not False, but the use of \u0027.durable\u0027 is a result of these tests evolving over time.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"},{"line_number":1627,"context_line":""},{"line_number":1628,"context_line":"    def test_hash_cleanup_listdir_reclaim(self):"},{"line_number":1629,"context_line":"        # Each scenario specifies a list of (filename, extension, [survives])"},{"line_number":1630,"context_line":"        # tuples. If extension is set or \u0027survives\u0027 is True, the filename"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_19260415","line":1627,"updated":"2016-09-07 12:04:21.000000000","message":"Is this actually a test for reclaim? (note that test name is now \"test_cleanup_ondisk_files_reclaim_with_data_files\") It looks a sort of a test for get_ondisk_files as similar with L1527.\n\nAh, it might be my mistake when working on rebasing though...","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"},{"line_number":1627,"context_line":""},{"line_number":1628,"context_line":"    def test_hash_cleanup_listdir_reclaim(self):"},{"line_number":1629,"context_line":"        # Each scenario specifies a list of (filename, extension, [survives])"},{"line_number":1630,"context_line":"        # tuples. If extension is set or \u0027survives\u0027 is True, the filename"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_a7831202","line":1627,"in_reply_to":"9a89bdaa_19260415","updated":"2016-09-13 18:49:29.000000000","message":"Rebase issue I think, this method should be named test_cleanup_ondisk_files_reclaim_with_data_files and line 1616 should be deleted. Will fix.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":1624,"context_line":"        ]"},{"line_number":1625,"context_line":"        self._test_get_ondisk_files(scenarios, POLICIES.default, frag_index\u003d1,"},{"line_number":1626,"context_line":"                                    frag_prefs\u003d[])"},{"line_number":1627,"context_line":""},{"line_number":1628,"context_line":"    def test_hash_cleanup_listdir_reclaim(self):"},{"line_number":1629,"context_line":"        # Each scenario specifies a list of (filename, extension, [survives])"},{"line_number":1630,"context_line":"        # tuples. If extension is set or \u0027survives\u0027 is True, the filename"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_abbb7571","line":1627,"in_reply_to":"9a89bdaa_a7831202","updated":"2016-09-14 10:28:27.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":4670,"context_line":"        self.assertEqual({ts_1: [0, 3], ts_3: [2]}, df.fragments)"},{"line_number":4671,"context_line":""},{"line_number":4672,"context_line":"        # with empty frag preferences we still get metadata from newer .meta"},{"line_number":4673,"context_line":"        # but applied to non-durable data file"},{"line_number":4674,"context_line":"        expected \u003d dict(frag_2_metadata)"},{"line_number":4675,"context_line":"        expected.update(meta_2_metadata)"},{"line_number":4676,"context_line":"        df \u003d df_mgr.get_diskfile(self.existing_device, \u00270\u0027, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027,"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_afb4ea81","line":4673,"updated":"2016-09-07 12:04:21.000000000","message":"Wonderful, I was feeling it might be unsafe but I didn\u0027t get the idea of the unsafe cases at all. Exactly, the newer meta file can be applied to the non-durable newer .data file. Nice.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":4670,"context_line":"        self.assertEqual({ts_1: [0, 3], ts_3: [2]}, df.fragments)"},{"line_number":4671,"context_line":""},{"line_number":4672,"context_line":"        # with empty frag preferences we still get metadata from newer .meta"},{"line_number":4673,"context_line":"        # but applied to non-durable data file"},{"line_number":4674,"context_line":"        expected \u003d dict(frag_2_metadata)"},{"line_number":4675,"context_line":"        expected.update(meta_2_metadata)"},{"line_number":4676,"context_line":"        df \u003d df_mgr.get_diskfile(self.existing_device, \u00270\u0027, \u0027a\u0027, \u0027c\u0027, \u0027o\u0027,"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_8797ee38","line":4673,"in_reply_to":"9a89bdaa_afb4ea81","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":4705,"context_line":"        self.assertEqual(ts_4.internal, df.timestamp)"},{"line_number":4706,"context_line":"        self.assertEqual(ts_1.internal, df.data_timestamp)"},{"line_number":4707,"context_line":"        self.assertEqual(ts_1.internal, df.durable_timestamp)"},{"line_number":4708,"context_line":"        self.assertEqual({ts_1.internal: [0, 3], ts_3.internal: [2]},"},{"line_number":4709,"context_line":"                         df.fragments)"},{"line_number":4710,"context_line":""},{"line_number":4711,"context_line":"        # prefer frags at ts_1, exclude frag index 3 so expect frag index 0"},{"line_number":4712,"context_line":"        prefs \u003d [{\u0027timestamp\u0027: ts_1.internal, \u0027exclude\u0027: [3]},"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_0b7149a8","line":4709,"range":{"start_line":4708,"start_character":25,"end_line":4709,"end_character":38},"updated":"2016-09-07 12:04:21.000000000","message":"I didn\u0027t get why the keys formatted as ts_1.internal (i.e. string)? the fragment property should return \"a dcit mapping \u003cTimestamp instance\u003e -\u003e \u003clist of frag indexes\u003e\"?\n\nAh, the reason it works is probably that assertEqual will compare between Timestamp instance and the string of timestamp.internal.\n\nProbably, any other tests need to be refactored likely\n\n self.assertEqual(ts_1.internal, df.durable_timestamp.internal)\n\nor, just\n\n self.assertEqual(ts_1, df.durable_timestamp)\n\n\nLooking at Timestamp impl[1], latter one seems also safe.\n\n1: https://github.com/openstack/swift/blob/master/swift/common/utils.py#L954-L959","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":4705,"context_line":"        self.assertEqual(ts_4.internal, df.timestamp)"},{"line_number":4706,"context_line":"        self.assertEqual(ts_1.internal, df.data_timestamp)"},{"line_number":4707,"context_line":"        self.assertEqual(ts_1.internal, df.durable_timestamp)"},{"line_number":4708,"context_line":"        self.assertEqual({ts_1.internal: [0, 3], ts_3.internal: [2]},"},{"line_number":4709,"context_line":"                         df.fragments)"},{"line_number":4710,"context_line":""},{"line_number":4711,"context_line":"        # prefer frags at ts_1, exclude frag index 3 so expect frag index 0"},{"line_number":4712,"context_line":"        prefs \u003d [{\u0027timestamp\u0027: ts_1.internal, \u0027exclude\u0027: [3]},"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_ca4d7b3f","line":4709,"range":{"start_line":4708,"start_character":25,"end_line":4709,"end_character":38},"in_reply_to":"9a89bdaa_0b7149a8","updated":"2016-09-13 18:49:29.000000000","message":"Good catch - should be ts_1: [0, 3] as for earlier assertions in the test.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":1582,"context_line":"                      (\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1583,"context_line":"                      (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027),"},{"line_number":1584,"context_line":"                      (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1585,"context_line":"                      (\u00270000000007.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1586,"context_line":""},{"line_number":1587,"context_line":"                     # specific frag older than tombstone is ignored"},{"line_number":1588,"context_line":"                     [(\u00270000000009.00000.ts\u0027, \u0027.ts\u0027),"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_0b4f09be","line":1585,"range":{"start_line":1585,"start_character":51,"end_line":1585,"end_character":61},"updated":"2016-09-14 10:37:45.000000000","message":"If we change the second item of the tuple for .durable file above (e.g. L1564) partially, maybe we should change all of them because right now the second item is not used for anything. Or we could reuse those .durable information to make assertions durable_frag_set in the scenario.\n\nPlease see my comment at L608 at patch set 39.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd672b6f8a8f2bd7f79dee08b81fc673ae55f1aa","unresolved":false,"context_lines":[{"line_number":1582,"context_line":"                      (\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1583,"context_line":"                      (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027),"},{"line_number":1584,"context_line":"                      (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1585,"context_line":"                      (\u00270000000007.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1586,"context_line":""},{"line_number":1587,"context_line":"                     # specific frag older than tombstone is ignored"},{"line_number":1588,"context_line":"                     [(\u00270000000009.00000.ts\u0027, \u0027.ts\u0027),"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_dd566729","line":1585,"range":{"start_line":1585,"start_character":51,"end_line":1585,"end_character":61},"in_reply_to":"7a8ec9b2_0b4f09be","updated":"2016-09-15 18:11:07.000000000","message":"The second item is used when calling self._test_cleanup_ondisk_files(scenarios, POLICIES.default) line 1615, it\u0027s just that it only needs to be True or False, but a value \u0027.durable\u0027 equates to True.\n\nI\u0027m not convinced (yet) about expanding the scope of _test_get_ondisk_files because that test is for the expected file keys in the method\u0027s returned dict. I think the \u0027durable_frag_set\u0027 key is implicitly tested in other places for example:\n\nhttps://github.com/openstack/swift/blob/5dc5f6d3d59a293f2048d5c446aa6996e61f765c/test/unit/obj/test_diskfile.py#L4623-L4623\n\nPerhaps we could discuss more as a follow up - if we do test for durable_frag_set in test_get_ondisk_files then I would prefer the expected value to be defined somehow in the test scenarios rather then inferred in the test e.g. second_item \u003d True means that the filename should give the durable timestamp for the diskfile???","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"e67a322b4ab5a46bfcbffee39e85e21716086328","unresolved":false,"context_lines":[{"line_number":1582,"context_line":"                      (\u00270000000007.00000#2.data\u0027, False, True),"},{"line_number":1583,"context_line":"                      (\u00270000000007.00000#1.data\u0027, \u0027.data\u0027),"},{"line_number":1584,"context_line":"                      (\u00270000000007.00000#0.data\u0027, False, True),"},{"line_number":1585,"context_line":"                      (\u00270000000007.00000.durable\u0027, \u0027.durable\u0027)],"},{"line_number":1586,"context_line":""},{"line_number":1587,"context_line":"                     # specific frag older than tombstone is ignored"},{"line_number":1588,"context_line":"                     [(\u00270000000009.00000.ts\u0027, \u0027.ts\u0027),"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_cd884e74","line":1585,"range":{"start_line":1585,"start_character":51,"end_line":1585,"end_character":61},"in_reply_to":"7a8ec9b2_dd566729","updated":"2016-09-16 11:25:18.000000000","message":"Ok, that is not urgent for me, let\u0027s discuss in the follow up. Done.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"}],"test/unit/obj/test_server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"ec1575a884c0a49ffb9b27f17a4dff1e5794ba44","unresolved":false,"context_lines":[{"line_number":1768,"context_line":"            # patch the commit method to do nothing so EC object gets"},{"line_number":1769,"context_line":"            # no .durable file"},{"line_number":1770,"context_line":"            with mock.patch(\u0027swift.obj.diskfile.ECDiskFileWriter.commit\u0027):"},{"line_number":1771,"context_line":"                resp \u003d req.get_response(self.object_controller)"},{"line_number":1772,"context_line":"            self.assertEqual(resp.status_int, 201)"},{"line_number":1773,"context_line":""},{"line_number":1774,"context_line":"            # Get the durable object"}],"source_content_type":"text/x-python","patch_set":5,"id":"ba15a1d1_ed5666d1","line":1771,"updated":"2015-09-15 22:45:13.000000000","message":"be sweet if we had a test that spawns a object server listener [1] so we could actually exercise the whole timeout/disconnect during the MIME body chunked request transfer...\n\n1. I think the ssync_receiver tests do this?","commit_id":"d2cff24c23bd76bc76cfcd54a0fecc435776b74c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":3128,"context_line":"                   \u0027Content-Length\u0027: \u00275\u0027,"},{"line_number":3129,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3130,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3131,"context_line":"        headers[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027] \u003d \u00270\u0027"},{"line_number":3132,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027,"},{"line_number":3133,"context_line":"                            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":3134,"context_line":"                            headers\u003dheaders)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_ae05cb7a","line":3131,"updated":"2016-09-07 12:04:21.000000000","message":"nits) it\u0027s not significant but this method can be called on Replication policy too so that it could be odd ondisk condition because replication policy file has Ec-Frag-Index...","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3128,"context_line":"                   \u0027Content-Length\u0027: \u00275\u0027,"},{"line_number":3129,"context_line":"                   \u0027Content-Type\u0027: \u0027application/octet-stream\u0027,"},{"line_number":3130,"context_line":"                   \u0027X-Backend-Storage-Policy-Index\u0027: int(policy)}"},{"line_number":3131,"context_line":"        headers[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027] \u003d \u00270\u0027"},{"line_number":3132,"context_line":"        req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027,"},{"line_number":3133,"context_line":"                            environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027PUT\u0027},"},{"line_number":3134,"context_line":"                            headers\u003dheaders)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_b6841ed3","line":3131,"in_reply_to":"9a89bdaa_ae05cb7a","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":3166,"context_line":""},{"line_number":3167,"context_line":"        return ts_0, ts_1, ts_2"},{"line_number":3168,"context_line":""},{"line_number":3169,"context_line":"    def test_GET_HEAD_with_fragment_preferences(self):"},{"line_number":3170,"context_line":"        for policy in POLICIES:"},{"line_number":3171,"context_line":"            ts_0, ts_1, ts_2 \u003d self._create_ondisk_fragments(policy)"},{"line_number":3172,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_4e019f37","line":3169,"updated":"2016-09-07 12:04:21.000000000","message":"it looks like this test includes both test cases with/without fragment preferences so it seems better to split the cases to \"with_fragment_preference\" and \"without_fragment_preference\" to be correct name?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":3166,"context_line":""},{"line_number":3167,"context_line":"        return ts_0, ts_1, ts_2"},{"line_number":3168,"context_line":""},{"line_number":3169,"context_line":"    def test_GET_HEAD_with_fragment_preferences(self):"},{"line_number":3170,"context_line":"        for policy in POLICIES:"},{"line_number":3171,"context_line":"            ts_0, ts_1, ts_2 \u003d self._create_ondisk_fragments(policy)"},{"line_number":3172,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_e6edc08f","line":3169,"in_reply_to":"9a89bdaa_21028ab4","updated":"2016-09-14 10:28:27.000000000","message":"ok, Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3166,"context_line":""},{"line_number":3167,"context_line":"        return ts_0, ts_1, ts_2"},{"line_number":3168,"context_line":""},{"line_number":3169,"context_line":"    def test_GET_HEAD_with_fragment_preferences(self):"},{"line_number":3170,"context_line":"        for policy in POLICIES:"},{"line_number":3171,"context_line":"            ts_0, ts_1, ts_2 \u003d self._create_ondisk_fragments(policy)"},{"line_number":3172,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_21028ab4","line":3169,"in_reply_to":"9a89bdaa_4e019f37","updated":"2016-09-13 18:49:29.000000000","message":"The cases without fragments are for sanity checking (and it seems I had that duplicated at start and end of test) so I would prefer to leave it as it is to avoid duplicating the test method just for the sanity check.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":3231,"context_line":"                _assert_repl_data_at_ts_2()"},{"line_number":3232,"context_line":"                self.assertEqual(resp.body, \u0027NEWER\u0027)"},{"line_number":3233,"context_line":""},{"line_number":3234,"context_line":"            req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027, headers\u003dheaders,"},{"line_number":3235,"context_line":"                                environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027HEAD\u0027})"},{"line_number":3236,"context_line":"            resp \u003d req.get_response(self.object_controller)"},{"line_number":3237,"context_line":""},{"line_number":3238,"context_line":"            if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3239,"context_line":"                _assert_frag_0_at_ts_0(resp)"},{"line_number":3240,"context_line":"            else:"},{"line_number":3241,"context_line":"                _assert_repl_data_at_ts_2()"},{"line_number":3242,"context_line":""},{"line_number":3243,"context_line":"            def _assert_frag_2_at_ts_2(resp):"},{"line_number":3244,"context_line":"                self.assertIn(resp.status_int, (200, 202))"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_ce1e8f6f","line":3241,"range":{"start_line":3234,"start_character":11,"end_line":3241,"end_character":43},"updated":"2016-09-07 12:04:21.000000000","message":"memo) those are for sanity.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3231,"context_line":"                _assert_repl_data_at_ts_2()"},{"line_number":3232,"context_line":"                self.assertEqual(resp.body, \u0027NEWER\u0027)"},{"line_number":3233,"context_line":""},{"line_number":3234,"context_line":"            req \u003d Request.blank(\u0027/sda1/p/a/c/o\u0027, headers\u003dheaders,"},{"line_number":3235,"context_line":"                                environ\u003d{\u0027REQUEST_METHOD\u0027: \u0027HEAD\u0027})"},{"line_number":3236,"context_line":"            resp \u003d req.get_response(self.object_controller)"},{"line_number":3237,"context_line":""},{"line_number":3238,"context_line":"            if policy.policy_type \u003d\u003d EC_POLICY:"},{"line_number":3239,"context_line":"                _assert_frag_0_at_ts_0(resp)"},{"line_number":3240,"context_line":"            else:"},{"line_number":3241,"context_line":"                _assert_repl_data_at_ts_2()"},{"line_number":3242,"context_line":""},{"line_number":3243,"context_line":"            def _assert_frag_2_at_ts_2(resp):"},{"line_number":3244,"context_line":"                self.assertIn(resp.status_int, (200, 202))"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_816356b0","line":3241,"range":{"start_line":3234,"start_character":11,"end_line":3241,"end_character":43},"in_reply_to":"9a89bdaa_ce1e8f6f","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"3769e8792ec1492e299d907a1cb65b9d7f54a202","unresolved":false,"context_lines":[{"line_number":3251,"context_line":"                    \u0027X-Backend-Fragments\u0027: backend_frags,"},{"line_number":3252,"context_line":"                    \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00272\u0027}"},{"line_number":3253,"context_line":"                self.assertDictContainsSubset(expect, resp.headers)"},{"line_number":3254,"context_line":"                self.assertNotIn(\u0027X-Object-Meta-Test\u0027, resp.headers)"},{"line_number":3255,"context_line":""},{"line_number":3256,"context_line":"            # Request with preferences can select the newer non-durable frag"},{"line_number":3257,"context_line":"            prefs \u003d json.dumps("}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_51406034","line":3254,"updated":"2016-09-07 12:04:21.000000000","message":"OK, good. with frag_prefs, older meta file will be ignored.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3251,"context_line":"                    \u0027X-Backend-Fragments\u0027: backend_frags,"},{"line_number":3252,"context_line":"                    \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: \u00272\u0027}"},{"line_number":3253,"context_line":"                self.assertDictContainsSubset(expect, resp.headers)"},{"line_number":3254,"context_line":"                self.assertNotIn(\u0027X-Object-Meta-Test\u0027, resp.headers)"},{"line_number":3255,"context_line":""},{"line_number":3256,"context_line":"            # Request with preferences can select the newer non-durable frag"},{"line_number":3257,"context_line":"            prefs \u003d json.dumps("}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_a1569acb","line":3254,"in_reply_to":"9a89bdaa_51406034","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"}],"test/unit/proxy/controllers/test_obj.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"08d8e05cdfbe276218241bf1082b2c911159f1cf","unresolved":false,"context_lines":[{"line_number":1751,"context_line":"                \u0027X-Object-Sysmeta-Ec-Content-Length\u0027: len(test_data),"},{"line_number":1752,"context_line":"                \u0027X-Object-Sysmeta-Ec-Etag\u0027: etag,"},{"line_number":1753,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: i,"},{"line_number":1754,"context_line":"                \u0027X-Timestamp\u0027: ts.normal,"},{"line_number":1755,"context_line":"                \u0027X-Backend-Timestamp\u0027: ts.internal"},{"line_number":1756,"context_line":"            }) for i, n in enumerate(primary_nodes)"},{"line_number":1757,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":29,"id":"5a5ae5dd_54846c09","line":1754,"updated":"2016-02-08 10:29:54.000000000","message":"X-Timestamp should be internal format, right?\n\nI noticed a bunch of unit tests which uses it with normal format though :/","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d11dc9a684ab8e8658e0e7c4fd6fc71c36f483e","unresolved":false,"context_lines":[{"line_number":1751,"context_line":"                \u0027X-Object-Sysmeta-Ec-Content-Length\u0027: len(test_data),"},{"line_number":1752,"context_line":"                \u0027X-Object-Sysmeta-Ec-Etag\u0027: etag,"},{"line_number":1753,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: i,"},{"line_number":1754,"context_line":"                \u0027X-Timestamp\u0027: ts.normal,"},{"line_number":1755,"context_line":"                \u0027X-Backend-Timestamp\u0027: ts.internal"},{"line_number":1756,"context_line":"            }) for i, n in enumerate(primary_nodes)"},{"line_number":1757,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":29,"id":"dae33548_c5c66a99","line":1754,"in_reply_to":"3a57f1b5_820e368a","updated":"2016-02-17 08:12:36.000000000","message":"Oh, yeah. Thanks.","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"9d9c4adbf1192ac1b4543f94b5e49d1cdee9036f","unresolved":false,"context_lines":[{"line_number":1751,"context_line":"                \u0027X-Object-Sysmeta-Ec-Content-Length\u0027: len(test_data),"},{"line_number":1752,"context_line":"                \u0027X-Object-Sysmeta-Ec-Etag\u0027: etag,"},{"line_number":1753,"context_line":"                \u0027X-Object-Sysmeta-Ec-Frag-Index\u0027: i,"},{"line_number":1754,"context_line":"                \u0027X-Timestamp\u0027: ts.normal,"},{"line_number":1755,"context_line":"                \u0027X-Backend-Timestamp\u0027: ts.internal"},{"line_number":1756,"context_line":"            }) for i, n in enumerate(primary_nodes)"},{"line_number":1757,"context_line":"        }"}],"source_content_type":"text/x-python","patch_set":29,"id":"3a57f1b5_820e368a","line":1754,"in_reply_to":"5a5ae5dd_54846c09","updated":"2016-02-12 18:21:33.000000000","message":"I think it is ts.normal:\n\nhttps://github.com/openstack/swift/blob/master/swift/obj/server.py#L774\n\nand X-Backend-Timestamp is the internal format","commit_id":"59e19a021d1dbb0d38ce3f2b6424ab91161e5770"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2216,"context_line":"                                    len(node_frags)))"},{"line_number":2217,"context_line":""},{"line_number":2218,"context_line":"            try:"},{"line_number":2219,"context_line":"                stub \u003d frags.pop(0)"},{"line_number":2220,"context_line":"            except IndexError:"},{"line_number":2221,"context_line":"                stub \u003d None"},{"line_number":2222,"context_line":"            if stub:"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_3a62a8d8","line":2219,"updated":"2016-09-08 10:36:43.000000000","message":"It\u0027s ok though, that might not look straight forward to someone (from randome choice to the first item) Maybe this is needed for some scenarios ordered by frag_prefs.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2216,"context_line":"                                    len(node_frags)))"},{"line_number":2217,"context_line":""},{"line_number":2218,"context_line":"            try:"},{"line_number":2219,"context_line":"                stub \u003d frags.pop(0)"},{"line_number":2220,"context_line":"            except IndexError:"},{"line_number":2221,"context_line":"                stub \u003d None"},{"line_number":2222,"context_line":"            if stub:"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_90655d46","line":2219,"in_reply_to":"9a89bdaa_3a62a8d8","updated":"2016-09-13 18:49:29.000000000","message":"On master, frags is always length of 1 so the random.choice was unnecessary.\n\nHere I use a list of frags to indicate the time order of different frag sets on the object server - first in the list is newest and so would be the first response expected, hence pop(0).\n\nBut, I noticed the pop() means that not all frags are used to construct x-backend-fragments on every call :/ So I am going to re-work this method and try to make it easier to understand at the same time.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2244,"context_line":"                    server._make_backend_fragments_header(alt_frags)"},{"line_number":2245,"context_line":"                frag_prefs \u003d req[\u0027headers\u0027].get("},{"line_number":2246,"context_line":"                    \u0027X-Backend-Fragment-Preferences\u0027)"},{"line_number":2247,"context_line":"                if not (frag_prefs or stub.get(\u0027durable\u0027, True)):"},{"line_number":2248,"context_line":"                    resp \u003d StubResponse(404)"},{"line_number":2249,"context_line":"                else:"},{"line_number":2250,"context_line":"                    resp \u003d StubResponse(200, body, headers)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_9ac01452","line":2247,"updated":"2016-09-08 10:36:43.000000000","message":"Ok, if frag_prefs exist, always true here. My wondering is that, since this commit, proxy-sever for EC policy always uses X-Backend-Fragment-Preference even if it\u0027s empty, right? If so, if frag_prefs doesn\u0027t exist, raise failure might be good?\n\nActually, [1] works. If you would want to emulate \"older\" object-server, anyway, that doesn\u0027t depend on the existence of x-backend-fragment-preferences here, IMO.\n\n1: https://gist.github.com/bloodeagle40234/0674611c0efae9e612e79724f836dcf6","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":2244,"context_line":"                    server._make_backend_fragments_header(alt_frags)"},{"line_number":2245,"context_line":"                frag_prefs \u003d req[\u0027headers\u0027].get("},{"line_number":2246,"context_line":"                    \u0027X-Backend-Fragment-Preferences\u0027)"},{"line_number":2247,"context_line":"                if not (frag_prefs or stub.get(\u0027durable\u0027, True)):"},{"line_number":2248,"context_line":"                    resp \u003d StubResponse(404)"},{"line_number":2249,"context_line":"                else:"},{"line_number":2250,"context_line":"                    resp \u003d StubResponse(200, body, headers)"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_e15daaed","line":2247,"in_reply_to":"9a89bdaa_35755b31","updated":"2016-09-14 10:28:27.000000000","message":"Make sense. That assertion failure may be captured in the green thread. That\u0027s worse. Ok, i don\u0027t have better way for now than emulating. Done.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2244,"context_line":"                    server._make_backend_fragments_header(alt_frags)"},{"line_number":2245,"context_line":"                frag_prefs \u003d req[\u0027headers\u0027].get("},{"line_number":2246,"context_line":"                    \u0027X-Backend-Fragment-Preferences\u0027)"},{"line_number":2247,"context_line":"                if not (frag_prefs or stub.get(\u0027durable\u0027, True)):"},{"line_number":2248,"context_line":"                    resp \u003d StubResponse(404)"},{"line_number":2249,"context_line":"                else:"},{"line_number":2250,"context_line":"                    resp \u003d StubResponse(200, body, headers)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_35755b31","line":2247,"in_reply_to":"9a89bdaa_9ac01452","updated":"2016-09-13 18:49:29.000000000","message":"I\u0027m not sure that raising an exception here if X-Backend-Fragment-Preferences did not exist would generate a useful test failure because the exception would be raised in a getter thread and probably caught by an \u0027except Exception\u0027. So i would prefer to leave this as it is - i.e. emulate the object server behaviour, do not assume the proxy is sending X-Backend-Fragment-Preferences, and have the tests discover a proxy bug by examining the expected response.\n\nIf the proxy server does not send X-Backend-Fragment-Preference then the existing unit tests will fail.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2309,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2310,"context_line":""},{"line_number":2311,"context_line":"    def test_GET_with_only_handoffs(self):"},{"line_number":2312,"context_line":"        obj2 \u003d self._make_ec_object_stub()"},{"line_number":2313,"context_line":""},{"line_number":2314,"context_line":"        node_frags \u003d [[{}]] * 14  # all primaries missing"},{"line_number":2315,"context_line":"        node_frags \u003d node_frags + [  # handoffs"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_15121757","line":2312,"updated":"2016-09-08 10:36:43.000000000","message":"Just a small question. Why is this \"obj2\" instead of \"obj\"?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2309,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2310,"context_line":""},{"line_number":2311,"context_line":"    def test_GET_with_only_handoffs(self):"},{"line_number":2312,"context_line":"        obj2 \u003d self._make_ec_object_stub()"},{"line_number":2313,"context_line":""},{"line_number":2314,"context_line":"        node_frags \u003d [[{}]] * 14  # all primaries missing"},{"line_number":2315,"context_line":"        node_frags \u003d node_frags + [  # handoffs"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f56f7328","line":2312,"in_reply_to":"9a89bdaa_15121757","updated":"2016-09-13 18:49:29.000000000","message":"Done. my mistake","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2351,"context_line":"        # ... regardless we should never need to fetch more than ec_ndata"},{"line_number":2352,"context_line":"        # frags for any given etag"},{"line_number":2353,"context_line":"        for etag, frags in collected_responses.items():"},{"line_number":2354,"context_line":"            self.assertTrue(len(frags) \u003c\u003d self.policy.ec_ndata,"},{"line_number":2355,"context_line":"                            \u0027collected %s frags for etag %s\u0027 % ("},{"line_number":2356,"context_line":"                                len(frags), etag))"},{"line_number":2357,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f5fcd3b9","line":2354,"updated":"2016-09-08 10:36:43.000000000","message":"len(frags) \u003c\u003d self.policy.ec_ndata seems doubtful a bit\nTo be more strict test case, should we assert the corrected_responses more obviously like:\n\n # only \u0027None\u0027 or obj2[\u0027etag\u0027] is in the log\n self.assertEqual(len(collected_responses), 2)\n # \u0027None\u0027 (i.e. 404s) doesn\u0027t have frag indexes\n self.assertEqual(len(collected_responses[None]), 1)\n # obj2[\u0027etag\u0027] has unique ec_ndata fragment indexes\n self.assertEqual(len(collected_responses[obj2[\u0027etag\u0027]]),\n                  self.policy.ec_ndata)\n\n???","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2351,"context_line":"        # ... regardless we should never need to fetch more than ec_ndata"},{"line_number":2352,"context_line":"        # frags for any given etag"},{"line_number":2353,"context_line":"        for etag, frags in collected_responses.items():"},{"line_number":2354,"context_line":"            self.assertTrue(len(frags) \u003c\u003d self.policy.ec_ndata,"},{"line_number":2355,"context_line":"                            \u0027collected %s frags for etag %s\u0027 % ("},{"line_number":2356,"context_line":"                                len(frags), etag))"},{"line_number":2357,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_b93ec5b5","line":2354,"in_reply_to":"9a89bdaa_f5fcd3b9","updated":"2016-09-13 18:49:29.000000000","message":"Agree\n\nI think this was cut and paste sanity checks from other tests","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2615,"context_line":""},{"line_number":2616,"context_line":"        # expect a request to all primaries plus one handoff"},{"line_number":2617,"context_line":"        self.assertEqual(self.replicas() + 1, len(log))"},{"line_number":2618,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2619,"context_line":"        for conn in log:"},{"line_number":2620,"context_line":"            fi \u003d conn.resp.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":2621,"context_line":"            if fi is not None:"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_1525b707","line":2618,"updated":"2016-09-08 10:36:43.000000000","message":"In test_GET_with_only_handoffs, collected_responses \u003d defaultdict(set).\n\nIs it better to adjust to either one?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2615,"context_line":""},{"line_number":2616,"context_line":"        # expect a request to all primaries plus one handoff"},{"line_number":2617,"context_line":"        self.assertEqual(self.replicas() + 1, len(log))"},{"line_number":2618,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2619,"context_line":"        for conn in log:"},{"line_number":2620,"context_line":"            fi \u003d conn.resp.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"},{"line_number":2621,"context_line":"            if fi is not None:"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_fe0d6138","line":2618,"in_reply_to":"9a89bdaa_1525b707","updated":"2016-09-13 18:49:29.000000000","message":"yes. list is better because we want to detect if any duplicate frag indexes are returned","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2627,"context_line":"        # proxy should ignore duplicated frag indexes and continue search for"},{"line_number":2628,"context_line":"        # a set of unique indexes, finding last one on a handoff"},{"line_number":2629,"context_line":"        node_frags \u003d ["},{"line_number":2630,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 0}, {\u0027obj\u0027: obj1, \u0027frag\u0027: 5}],"},{"line_number":2631,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 0},  # duplicate frag"},{"line_number":2632,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 1},"},{"line_number":2633,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 1},  # duplicate frag"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f570530d","line":2630,"updated":"2016-09-08 10:36:43.000000000","message":"Yey! It will happen during replication!","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2627,"context_line":"        # proxy should ignore duplicated frag indexes and continue search for"},{"line_number":2628,"context_line":"        # a set of unique indexes, finding last one on a handoff"},{"line_number":2629,"context_line":"        node_frags \u003d ["},{"line_number":2630,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 0}, {\u0027obj\u0027: obj1, \u0027frag\u0027: 5}],"},{"line_number":2631,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 0},  # duplicate frag"},{"line_number":2632,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 1},"},{"line_number":2633,"context_line":"            {\u0027obj\u0027: obj1, \u0027frag\u0027: 1},  # duplicate frag"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_be783991","line":2630,"in_reply_to":"9a89bdaa_f570530d","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2657,"context_line":"        # request to node 1. Actual value could be less if the extra request"},{"line_number":2658,"context_line":"        # occurs and quorum is reached before requests to nodes with a"},{"line_number":2659,"context_line":"        # duplicate frag."},{"line_number":2660,"context_line":"        self.assertLessEqual(len(log), self.replicas() + 1)"},{"line_number":2661,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2662,"context_line":"        for conn in log:"},{"line_number":2663,"context_line":"            fi \u003d conn.resp.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_d5ea6f73","line":2660,"updated":"2016-09-08 10:36:43.000000000","message":"Yes, this means we can connect to a same node twice to retrieve same object with different index in the new behavior.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2657,"context_line":"        # request to node 1. Actual value could be less if the extra request"},{"line_number":2658,"context_line":"        # occurs and quorum is reached before requests to nodes with a"},{"line_number":2659,"context_line":"        # duplicate frag."},{"line_number":2660,"context_line":"        self.assertLessEqual(len(log), self.replicas() + 1)"},{"line_number":2661,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2662,"context_line":"        for conn in log:"},{"line_number":2663,"context_line":"            fi \u003d conn.resp.headers.get(\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027)"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_fece41b3","line":2660,"in_reply_to":"9a89bdaa_d5ea6f73","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2730,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2731,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2732,"context_line":""},{"line_number":2733,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2734,"context_line":"        # never get a quorum so all nodes are searched"},{"line_number":2735,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2736,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_99fca1b9","line":2733,"updated":"2016-09-08 10:36:43.000000000","message":"OK. All available buckets are shortfall and bad_bucket is empty. And then, no statuses will make HTTPServiceUnavaiable  in the best_response[1].\n\n1: https://github.com/openstack/swift/blob/master/swift/proxy/controllers/base.py#L1689-L1690","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2730,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2731,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2732,"context_line":""},{"line_number":2733,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2734,"context_line":"        # never get a quorum so all nodes are searched"},{"line_number":2735,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2736,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_5ebe5500","line":2733,"in_reply_to":"9a89bdaa_99fca1b9","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2732,"context_line":""},{"line_number":2733,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2734,"context_line":"        # never get a quorum so all nodes are searched"},{"line_number":2735,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2736,"context_line":""},{"line_number":2737,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2738,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_d9c70962","line":2735,"updated":"2016-09-08 10:36:43.000000000","message":"Why not we don\u0027t check the collected_responses as well as other tests?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2732,"context_line":""},{"line_number":2733,"context_line":"        self.assertEqual(resp.status_int, 503)"},{"line_number":2734,"context_line":"        # never get a quorum so all nodes are searched"},{"line_number":2735,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"},{"line_number":2736,"context_line":""},{"line_number":2737,"context_line":"    def test_GET_with_mixed_frags_and_no_quorum_will_503(self):"},{"line_number":2738,"context_line":"        # all nodes have a frag but there is no one set that reaches quorum,"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_bef2f96d","line":2735,"in_reply_to":"9a89bdaa_d9c70962","updated":"2016-09-13 18:49:29.000000000","message":"I\u0027ll add checks. These are really just sanity checks that the test framework is returning the expected response.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2787,"context_line":"        for conn in log:"},{"line_number":2788,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2789,"context_line":"            collected_etags.add(etag)"},{"line_number":2790,"context_line":"            collected_status.add(conn.resp.status)"},{"line_number":2791,"context_line":""},{"line_number":2792,"context_line":"        # default node_iter will exhaust at 2 * replicas"},{"line_number":2793,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_99cd8168","line":2790,"updated":"2016-09-08 10:36:43.000000000","message":"Oh, here, we use another version, collected_etags and collected_status instead of neither collected_responses nor collected_indexes.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":2787,"context_line":"        for conn in log:"},{"line_number":2788,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2789,"context_line":"            collected_etags.add(etag)"},{"line_number":2790,"context_line":"            collected_status.add(conn.resp.status)"},{"line_number":2791,"context_line":""},{"line_number":2792,"context_line":"        # default node_iter will exhaust at 2 * replicas"},{"line_number":2793,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_c1f98e4f","line":2790,"in_reply_to":"7a8ec9b2_3e86e9b9","updated":"2016-09-14 10:28:27.000000000","message":"Ok, it\u0027s just a task for house keeping. Not urgent, IMO.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2787,"context_line":"        for conn in log:"},{"line_number":2788,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2789,"context_line":"            collected_etags.add(etag)"},{"line_number":2790,"context_line":"            collected_status.add(conn.resp.status)"},{"line_number":2791,"context_line":""},{"line_number":2792,"context_line":"        # default node_iter will exhaust at 2 * replicas"},{"line_number":2793,"context_line":"        self.assertEqual(len(log), 2 * self.replicas())"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_3e86e9b9","line":2790,"in_reply_to":"9a89bdaa_99cd8168","updated":"2016-09-13 18:49:29.000000000","message":"hmmm, I *think* I just put these checks in as sanity checks and i agree they are not all consistent.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2835,"context_line":"        # because nodes are shuffled we can\u0027t be sure how many durables are"},{"line_number":2836,"context_line":"        # returned but it must be at least 1"},{"line_number":2837,"context_line":"        self.assertLessEqual(len(collected_durables),"},{"line_number":2838,"context_line":"                             self.policy.ec_nparity + 1)"},{"line_number":2839,"context_line":"        self.assertGreaterEqual(len(collected_durables), 1)"},{"line_number":2840,"context_line":""},{"line_number":2841,"context_line":"    def test_GET_with_single_durable_file(self):"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_3950f531","line":2838,"updated":"2016-09-08 10:36:43.000000000","message":"exactly we set durable\u003dTrue for 5 fragments but the fixed value seems better here, right? That doesn\u0027t seem to depend on the number of parities but depend on the test setting.\n\ni.e.:\n self.assertLessEqual(len(collected_durables), 5)","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2835,"context_line":"        # because nodes are shuffled we can\u0027t be sure how many durables are"},{"line_number":2836,"context_line":"        # returned but it must be at least 1"},{"line_number":2837,"context_line":"        self.assertLessEqual(len(collected_durables),"},{"line_number":2838,"context_line":"                             self.policy.ec_nparity + 1)"},{"line_number":2839,"context_line":"        self.assertGreaterEqual(len(collected_durables), 1)"},{"line_number":2840,"context_line":""},{"line_number":2841,"context_line":"    def test_GET_with_single_durable_file(self):"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_be4e7994","line":2838,"in_reply_to":"9a89bdaa_3950f531","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2877,"context_line":"                collected_durables.append(conn)"},{"line_number":2878,"context_line":"        # because nodes are shuffled we can\u0027t be sure how many non-durables"},{"line_number":2879,"context_line":"        # are returned before the durable, but we do expect a single durable"},{"line_number":2880,"context_line":"        self.assertEqual(1, len(collected_durables))"},{"line_number":2881,"context_line":""},{"line_number":2882,"context_line":"    def test_GET_with_no_durable_files(self):"},{"line_number":2883,"context_line":"        # verify that at least one durable is necessary for a successful GET"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f9004d04","line":2880,"updated":"2016-09-08 10:36:43.000000000","message":"self.assertLessEqual(len(collected_durables), 14)\n\nfor sanity?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2877,"context_line":"                collected_durables.append(conn)"},{"line_number":2878,"context_line":"        # because nodes are shuffled we can\u0027t be sure how many non-durables"},{"line_number":2879,"context_line":"        # are returned before the durable, but we do expect a single durable"},{"line_number":2880,"context_line":"        self.assertEqual(1, len(collected_durables))"},{"line_number":2881,"context_line":""},{"line_number":2882,"context_line":"    def test_GET_with_no_durable_files(self):"},{"line_number":2883,"context_line":"        # verify that at least one durable is necessary for a successful GET"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_a12578af","line":2880,"in_reply_to":"9a89bdaa_f9004d04","updated":"2016-09-13 18:49:29.000000000","message":"we are already asserting it is equal to exactly 1","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2905,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2906,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2907,"context_line":""},{"line_number":2908,"context_line":"        self.assertEqual(resp.status_int, 404)"},{"line_number":2909,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":2910,"context_line":"        # report having a durable timestamp"},{"line_number":2911,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_5c1f675a","line":2908,"updated":"2016-09-08 10:36:43.000000000","message":"Nice","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2905,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2906,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2907,"context_line":""},{"line_number":2908,"context_line":"        self.assertEqual(resp.status_int, 404)"},{"line_number":2909,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":2910,"context_line":"        # report having a durable timestamp"},{"line_number":2911,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_e11f8059","line":2908,"in_reply_to":"9a89bdaa_5c1f675a","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2993,"context_line":""},{"line_number":2994,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":2995,"context_line":"        self.assertEqual(resp.headers[\u0027etag\u0027], obj1[\u0027etag\u0027])"},{"line_number":2996,"context_line":"        self.assertEqual(md5(resp.body).hexdigest(), obj1[\u0027etag\u0027])"},{"line_number":2997,"context_line":""},{"line_number":2998,"context_line":"    def test_GET_with_missing_durables_and_older_durables(self):"},{"line_number":2999,"context_line":"        # scenario: non-durable frags of newer obj1 obscure all durable frags"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_fc5cbb1a","line":2996,"updated":"2016-09-08 10:36:43.000000000","message":"Awesome, optimistic-GET never give up!","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2993,"context_line":""},{"line_number":2994,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":2995,"context_line":"        self.assertEqual(resp.headers[\u0027etag\u0027], obj1[\u0027etag\u0027])"},{"line_number":2996,"context_line":"        self.assertEqual(md5(resp.body).hexdigest(), obj1[\u0027etag\u0027])"},{"line_number":2997,"context_line":""},{"line_number":2998,"context_line":"    def test_GET_with_missing_durables_and_older_durables(self):"},{"line_number":2999,"context_line":"        # scenario: non-durable frags of newer obj1 obscure all durable frags"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_01254caa","line":2996,"in_reply_to":"9a89bdaa_fc5cbb1a","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":2999,"context_line":"        # scenario: non-durable frags of newer obj1 obscure all durable frags"},{"line_number":3000,"context_line":"        # of older obj2, so first 14 requests result in a non-durable set."},{"line_number":3001,"context_line":"        # At that point (or before) the proxy knows that a durable set of"},{"line_number":3002,"context_line":"        # frags for obj1 exists so will fetch them, requiring another 10"},{"line_number":3003,"context_line":"        # directed requests."},{"line_number":3004,"context_line":"        obj2 \u003d self._make_ec_object_stub(timestamp\u003dself._ts_iter.next())"},{"line_number":3005,"context_line":"        obj1 \u003d self._make_ec_object_stub(timestamp\u003dself._ts_iter.next())"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_c5fb9777","line":3002,"range":{"start_line":3002,"start_character":20,"end_line":3002,"end_character":24},"updated":"2016-09-08 10:36:43.000000000","message":"obj2?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":2999,"context_line":"        # scenario: non-durable frags of newer obj1 obscure all durable frags"},{"line_number":3000,"context_line":"        # of older obj2, so first 14 requests result in a non-durable set."},{"line_number":3001,"context_line":"        # At that point (or before) the proxy knows that a durable set of"},{"line_number":3002,"context_line":"        # frags for obj1 exists so will fetch them, requiring another 10"},{"line_number":3003,"context_line":"        # directed requests."},{"line_number":3004,"context_line":"        obj2 \u003d self._make_ec_object_stub(timestamp\u003dself._ts_iter.next())"},{"line_number":3005,"context_line":"        obj1 \u003d self._make_ec_object_stub(timestamp\u003dself._ts_iter.next())"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_a173b8b3","line":3002,"range":{"start_line":3002,"start_character":20,"end_line":3002,"end_character":24},"in_reply_to":"9a89bdaa_c5fb9777","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":3029,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False}],  # obj2 missing"},{"line_number":3030,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},"},{"line_number":3031,"context_line":"             {\u0027obj\u0027: obj2, \u0027frag\u0027: 13, \u0027durable\u0027: True}],"},{"line_number":3032,"context_line":"        ] + [[{}]] * self.replicas()  # handoffs"},{"line_number":3033,"context_line":""},{"line_number":3034,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3035,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_7cdfcb39","line":3032,"range":{"start_line":3032,"start_character":10,"end_line":3032,"end_character":48},"updated":"2016-09-08 10:36:43.000000000","message":"we don\u0027t need handoffs in this scenario because primaries have enough fragments to return 200.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3029,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False}],  # obj2 missing"},{"line_number":3030,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},"},{"line_number":3031,"context_line":"             {\u0027obj\u0027: obj2, \u0027frag\u0027: 13, \u0027durable\u0027: True}],"},{"line_number":3032,"context_line":"        ] + [[{}]] * self.replicas()  # handoffs"},{"line_number":3033,"context_line":""},{"line_number":3034,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3035,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_812cfcac","line":3032,"range":{"start_line":3032,"start_character":10,"end_line":3032,"end_character":48},"in_reply_to":"9a89bdaa_7cdfcb39","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":3072,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 11, \u0027durable\u0027: False}],"},{"line_number":3073,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 12, \u0027durable\u0027: False}],"},{"line_number":3074,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 13, \u0027durable\u0027: False}],"},{"line_number":3075,"context_line":"        ] + [[{}]] * self.replicas()  # handoffs"},{"line_number":3076,"context_line":""},{"line_number":3077,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3078,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f84afe53","line":3075,"range":{"start_line":3075,"start_character":9,"end_line":3075,"end_character":48},"updated":"2016-09-08 10:36:43.000000000","message":"This is unnecessary too?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3072,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 11, \u0027durable\u0027: False}],"},{"line_number":3073,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 12, \u0027durable\u0027: False}],"},{"line_number":3074,"context_line":"            [{\u0027obj\u0027: obj3, \u0027frag\u0027: 13, \u0027durable\u0027: False}],"},{"line_number":3075,"context_line":"        ] + [[{}]] * self.replicas()  # handoffs"},{"line_number":3076,"context_line":""},{"line_number":3077,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3078,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_61061022","line":3075,"range":{"start_line":3075,"start_character":9,"end_line":3075,"end_character":48},"in_reply_to":"9a89bdaa_f84afe53","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":3118,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False}],  # obj2 missing"},{"line_number":3119,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},"},{"line_number":3120,"context_line":"             {\u0027obj\u0027: obj2, \u0027frag\u0027: 13, \u0027durable\u0027: True}],"},{"line_number":3121,"context_line":"        ] + [[{}]] * (self.replicas() + 1)  # 1 empty primary plus handoffs"},{"line_number":3122,"context_line":""},{"line_number":3123,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3124,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_58c04a50","line":3121,"range":{"start_line":3121,"start_character":10,"end_line":3121,"end_character":75},"updated":"2016-09-08 10:36:43.000000000","message":"ditto. the reason why we don\u0027t need it is maybe available fragments will be served by node_provider prior to handoffs?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":3118,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False}],  # obj2 missing"},{"line_number":3119,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},"},{"line_number":3120,"context_line":"             {\u0027obj\u0027: obj2, \u0027frag\u0027: 13, \u0027durable\u0027: True}],"},{"line_number":3121,"context_line":"        ] + [[{}]] * (self.replicas() + 1)  # 1 empty primary plus handoffs"},{"line_number":3122,"context_line":""},{"line_number":3123,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3124,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_5320f59b","line":3121,"range":{"start_line":3121,"start_character":10,"end_line":3121,"end_character":75},"in_reply_to":"7a8ec9b2_84c5ea5c","updated":"2016-09-14 10:28:27.000000000","message":"OK, done.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3118,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 12, \u0027durable\u0027: False}],  # obj2 missing"},{"line_number":3119,"context_line":"            [{\u0027obj\u0027: obj1, \u0027frag\u0027: 13, \u0027durable\u0027: False},"},{"line_number":3120,"context_line":"             {\u0027obj\u0027: obj2, \u0027frag\u0027: 13, \u0027durable\u0027: True}],"},{"line_number":3121,"context_line":"        ] + [[{}]] * (self.replicas() + 1)  # 1 empty primary plus handoffs"},{"line_number":3122,"context_line":""},{"line_number":3123,"context_line":"        fake_response \u003d self._fake_ec_node_response(list(node_frags))"},{"line_number":3124,"context_line":""}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_84c5ea5c","line":3121,"range":{"start_line":3121,"start_character":10,"end_line":3121,"end_character":75},"in_reply_to":"9a89bdaa_58c04a50","updated":"2016-09-13 18:49:29.000000000","message":"yes, agree.\n\nevery first response from a primary informs the proxy of an alternate fragment (so node_provider can feed an alternate node instead of a handoff being used) AND the proxy does not fire off new requests while there are more pending requests than the shortfall, so all first primary requests must respond before proxy fires off more requests, and at this time there are alternate nodes available and so handoffs are not used.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":3620,"context_line":"        self.assertEqual(len(node_fragments), self.replicas())  # sanity"},{"line_number":3621,"context_line":"        headers \u003d {\u0027X-Object-Sysmeta-Ec-Content-Length\u0027: str(len(real_body)),"},{"line_number":3622,"context_line":"                   \u0027X-Object-Sysmeta-Ec-Etag\u0027: body_etag,"},{"line_number":3623,"context_line":"                   \u0027X-Backend-Durable\u0027: \u0027True\u0027}"},{"line_number":3624,"context_line":"        start \u003d int(req_range.split(\u0027-\u0027)[0])"},{"line_number":3625,"context_line":"        self.assertGreaterEqual(start, 0)  # sanity"},{"line_number":3626,"context_line":"        title, exp \u003d swob.RESPONSE_REASONS[416]"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f892fef4","line":3623,"range":{"start_line":3623,"start_character":18,"end_line":3623,"end_character":47},"updated":"2016-09-08 10:36:43.000000000","message":"What\u0027s that actually? X-Backend-Durable-Timestamp?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":3620,"context_line":"        self.assertEqual(len(node_fragments), self.replicas())  # sanity"},{"line_number":3621,"context_line":"        headers \u003d {\u0027X-Object-Sysmeta-Ec-Content-Length\u0027: str(len(real_body)),"},{"line_number":3622,"context_line":"                   \u0027X-Object-Sysmeta-Ec-Etag\u0027: body_etag,"},{"line_number":3623,"context_line":"                   \u0027X-Backend-Durable\u0027: \u0027True\u0027}"},{"line_number":3624,"context_line":"        start \u003d int(req_range.split(\u0027-\u0027)[0])"},{"line_number":3625,"context_line":"        self.assertGreaterEqual(start, 0)  # sanity"},{"line_number":3626,"context_line":"        title, exp \u003d swob.RESPONSE_REASONS[416]"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_04b55acc","line":3623,"range":{"start_line":3623,"start_character":18,"end_line":3623,"end_character":47},"in_reply_to":"9a89bdaa_f892fef4","updated":"2016-09-13 18:49:29.000000000","message":"Done. mistake.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":2423,"context_line":"        self.assertEqual(resp.headers[\u0027etag\u0027], obj1[\u0027etag\u0027])"},{"line_number":2424,"context_line":"        self.assertEqual(md5(resp.body).hexdigest(), obj1[\u0027etag\u0027])"},{"line_number":2425,"context_line":""},{"line_number":2426,"context_line":"        collected_responses \u003d defaultdict(list)"},{"line_number":2427,"context_line":"        for conn in log:"},{"line_number":2428,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2429,"context_line":"            index \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Frag-Index\u0027]"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_4103be73","line":2426,"range":{"start_line":2426,"start_character":42,"end_line":2426,"end_character":46},"updated":"2016-09-14 10:37:45.000000000","message":"nice","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"72bb572026d960823a2137758e3592b667c4910f","unresolved":false,"context_lines":[{"line_number":2778,"context_line":"        # expect a request to all nodes"},{"line_number":2779,"context_line":"        self.assertEqual(2 * self.replicas(), len(log))"},{"line_number":2780,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2781,"context_line":"        collected_etags \u003d set()"},{"line_number":2782,"context_line":"        for conn in log:"},{"line_number":2783,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2784,"context_line":"            collected_etags.add(etag)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_415f3e72","line":2781,"updated":"2016-09-14 10:37:45.000000000","message":"collected_etags is defined, and each etag is added to collected_etags. *BUT NEVER ASSERTED*\n\n if etag is not None:\n   collected_etags.add(etag)\n\nat L2784 and\n\n self.assertEqual(len(collected_etags), 1)\n\nat L2788 are needed.","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"fd672b6f8a8f2bd7f79dee08b81fc673ae55f1aa","unresolved":false,"context_lines":[{"line_number":2778,"context_line":"        # expect a request to all nodes"},{"line_number":2779,"context_line":"        self.assertEqual(2 * self.replicas(), len(log))"},{"line_number":2780,"context_line":"        collected_indexes \u003d defaultdict(list)"},{"line_number":2781,"context_line":"        collected_etags \u003d set()"},{"line_number":2782,"context_line":"        for conn in log:"},{"line_number":2783,"context_line":"            etag \u003d conn.resp.headers[\u0027X-Object-Sysmeta-Ec-Etag\u0027]"},{"line_number":2784,"context_line":"            collected_etags.add(etag)"}],"source_content_type":"text/x-python","patch_set":40,"id":"7a8ec9b2_7dae3399","line":2781,"in_reply_to":"7a8ec9b2_415f3e72","updated":"2016-09-15 18:11:07.000000000","message":"Done","commit_id":"5dc5f6d3d59a293f2048d5c446aa6996e61f765c"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"a78bc4bc91d294bf02f9e5e26dec5ff91076b0df","unresolved":false,"context_lines":[{"line_number":2997,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":2998,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":2999,"context_line":""},{"line_number":3000,"context_line":"        self.assertEqual(resp.status_int, 404)"},{"line_number":3001,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":3002,"context_line":"        # report having a durable timestamp"},{"line_number":3003,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":41,"id":"7a8ec9b2_c66cc1d2","line":3000,"updated":"2016-09-16 03:43:43.000000000","message":"I think this is actively codifying a behavior we don\u0027t want.\n\nIf you imagine the previous test is sane, rather than imaging frag #0 (the only durable above) as NOT durable - instead imagine it is just unavailable.\n\nWe have all of the frags (more than enough really) to rebuild the object and instead of doing so we\u0027re returning a 404 because we don\u0027t trust our durability engine?  That\u0027s nuts.\n\nIf we think this situation is common enough to consider - we should codify the behavior we want (high availability) and file a bug against the reconstructor for not fixing it\u0027s shiz (high durability).","commit_id":"8f5755b8ac962d1a8e3a0cae1d257de45eaab499"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"422d3c5f652c5fac3263e027fd844209a5c1115a","unresolved":false,"context_lines":[{"line_number":2227,"context_line":"        policy \u003d policy or self.policy"},{"line_number":2228,"context_line":"        segment_size \u003d policy.ec_segment_size"},{"line_number":2229,"context_line":"        test_body \u003d test_body or ("},{"line_number":2230,"context_line":"            \u0027test\u0027 * segment_size)[:-random.randint(1, 1000)]"},{"line_number":2231,"context_line":"        timestamp \u003d timestamp or Timestamp(time.time())"},{"line_number":2232,"context_line":"        etag \u003d md5(test_body).hexdigest()"},{"line_number":2233,"context_line":"        ec_archive_bodies \u003d self._make_ec_archive_bodies(test_body,"}],"source_content_type":"text/x-python","patch_set":43,"id":"7a8ec9b2_5c4796bb","line":2230,"updated":"2016-09-16 11:22:19.000000000","message":"Yes! we need this change because string[:0] will make an empty string (i.e. \u0027\u0027) and that causes no ec_achive_bodies which causes failures.\n\nThe snippet is here, https://gist.github.com/bloodeagle40234/b9c3677fddd7deecb7efcc96a42b5025","commit_id":"44a861787a60346aded8be2a54ef4e16accccbb6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4d5ba8bc6ae755ecc9643a85e411a951c9155f73","unresolved":false,"context_lines":[{"line_number":2227,"context_line":"        policy \u003d policy or self.policy"},{"line_number":2228,"context_line":"        segment_size \u003d policy.ec_segment_size"},{"line_number":2229,"context_line":"        test_body \u003d test_body or ("},{"line_number":2230,"context_line":"            \u0027test\u0027 * segment_size)[:-random.randint(1, 1000)]"},{"line_number":2231,"context_line":"        timestamp \u003d timestamp or Timestamp(time.time())"},{"line_number":2232,"context_line":"        etag \u003d md5(test_body).hexdigest()"},{"line_number":2233,"context_line":"        ec_archive_bodies \u003d self._make_ec_archive_bodies(test_body,"}],"source_content_type":"text/x-python","patch_set":43,"id":"7a8ec9b2_cc275eb4","line":2230,"in_reply_to":"7a8ec9b2_5c4796bb","updated":"2016-09-16 20:19:24.000000000","message":"Oh dear, because -0 is not a thing!","commit_id":"44a861787a60346aded8be2a54ef4e16accccbb6"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"4d5ba8bc6ae755ecc9643a85e411a951c9155f73","unresolved":false,"context_lines":[{"line_number":3002,"context_line":"        with capture_http_requests(fake_response) as log:"},{"line_number":3003,"context_line":"            resp \u003d req.get_response(self.app)"},{"line_number":3004,"context_line":""},{"line_number":3005,"context_line":"        self.assertEqual(resp.status_int, 404)"},{"line_number":3006,"context_line":"        # all 28 nodes tried with an optimistic get, none are durable and none"},{"line_number":3007,"context_line":"        # report having a durable timestamp"},{"line_number":3008,"context_line":"        self.assertEqual(28, len(log))"}],"source_content_type":"text/x-python","patch_set":43,"id":"7a8ec9b2_0dc93a59","line":3005,"updated":"2016-09-16 20:19:24.000000000","message":"I sill think this is highlighting an undesirable behavior - but it\u0027s an existing issues that\u0027s made no worse; hardly a regression - I\u0027ll write up another bug report - it\u0027ll be interesting (to me) to see if the fix is to remove code that was doing the special case handling.","commit_id":"44a861787a60346aded8be2a54ef4e16accccbb6"}],"test/unit/proxy/test_server.py":[{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":6101,"context_line":"         obj2srv, obj3srv) \u003d _test_servers"},{"line_number":6102,"context_line":"        ec_policy \u003d POLICIES[3]"},{"line_number":6103,"context_line":"        container_name \u003d obj_name \u003d uuid.uuid4().hex"},{"line_number":6104,"context_line":"        obj_name \u003d uuid.uuid4().hex"},{"line_number":6105,"context_line":"        obj_path \u003d os.path.join(os.sep, \u0027v1\u0027, \u0027a\u0027, container_name, obj_name)"},{"line_number":6106,"context_line":""},{"line_number":6107,"context_line":"        # PUT container, make sure it worked"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_72d663f2","line":6104,"updated":"2015-09-30 12:56:04.000000000","message":"obj_name seems already set 1 line before as uuid.uuid4().hex.\n\nDid you intend to set another name to obj_name?","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":6101,"context_line":"         obj2srv, obj3srv) \u003d _test_servers"},{"line_number":6102,"context_line":"        ec_policy \u003d POLICIES[3]"},{"line_number":6103,"context_line":"        container_name \u003d obj_name \u003d uuid.uuid4().hex"},{"line_number":6104,"context_line":"        obj_name \u003d uuid.uuid4().hex"},{"line_number":6105,"context_line":"        obj_path \u003d os.path.join(os.sep, \u0027v1\u0027, \u0027a\u0027, container_name, obj_name)"},{"line_number":6106,"context_line":""},{"line_number":6107,"context_line":"        # PUT container, make sure it worked"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_f6d9ecf3","line":6104,"in_reply_to":"9a1a9d01_72d663f2","updated":"2015-09-30 23:33:45.000000000","message":"oops! thanks","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"312745b1b4ed560d0989048ea775f2c4afa55d17","unresolved":false,"context_lines":[{"line_number":6239,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d2, exts\u003d(\u0027.data\u0027,))],"},{"line_number":6240,"context_line":"            2: []"},{"line_number":6241,"context_line":"        }"},{"line_number":6242,"context_line":""},{"line_number":6243,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":6244,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":6245,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_840e4c8a","line":6242,"updated":"2015-09-30 12:56:04.000000000","message":"To be honest, I wanna restrict # of fragments per object because our ec engine \"SHSS\" ensures the security by assuming \u003e\u003dk fragments never  in a device but maybe it would be future work.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"ab98394521de2453af5232de14c28c5bc34810ff","unresolved":false,"context_lines":[{"line_number":6239,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d2, exts\u003d(\u0027.data\u0027,))],"},{"line_number":6240,"context_line":"            2: []"},{"line_number":6241,"context_line":"        }"},{"line_number":6242,"context_line":""},{"line_number":6243,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":6244,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":6245,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_f6232cbb","line":6242,"in_reply_to":"9a1a9d01_840e4c8a","updated":"2015-09-30 23:33:45.000000000","message":"Do you mean restrict # fragments per *node*? That would be a separate piece of work though, to change the reconstructor?\n\nAdmittedly this scenario is *unlikely* but could happen after rebalances I think.","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"a2b85f673883fe9ed07f695e6d7d683f35fb845a","unresolved":false,"context_lines":[{"line_number":6239,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d2, exts\u003d(\u0027.data\u0027,))],"},{"line_number":6240,"context_line":"            2: []"},{"line_number":6241,"context_line":"        }"},{"line_number":6242,"context_line":""},{"line_number":6243,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":6244,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":6245,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"}],"source_content_type":"text/x-python","patch_set":16,"id":"9a1a9d01_2f721d4a","line":6242,"in_reply_to":"9a1a9d01_f6232cbb","updated":"2015-10-05 07:08:59.000000000","message":"Exactly, that might be another work, sorry. This seems to focus on the behavior AFTER swift placed multiple fragments in a disk in rebalancing scenario.\n\n\nFor your information, I wrote down what I wanted to tell here as follows:\n\nI meant, # fragments per disk. (I thought, swift means \"a node\" as \"a disk\") And our SHSS has a model to prevent security risks for failure disk replacement like \"send back to HDD venders\". Currently if no enough fragments (i.e. \u003e\u003dk) in A DISK, no information is possible to be leaked.\n\nSo I meant I\u0027d like to maintain the condition of multiple fragments in a disk like as \n\n /srv/node/sdb/objects/part/suffix/hash/ts#1.data\n /srv/node/sdb/objects/part/suffix/hash/ts#2.data\n /srv/node/sdb/objects/part/suffix/hash/ts.durable\n\nin rebalancing scenario.\n\nIt looks like as reconstructor work, exactly :\\","commit_id":"0862e2df543237cd277978c8ee3c34647a02ba78"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":5598,"context_line":"    def _setup_nodes_and_do_GET(self, objs, node_state):"},{"line_number":5599,"context_line":"        \"\"\""},{"line_number":5600,"context_line":"        A helper method that creates object fragments, stashes them in temp"},{"line_number":5601,"context_line":"        dirs, and then moves selected frgs back into the hash_dirs on each node"},{"line_number":5602,"context_line":"        according to a specified desired node state description."},{"line_number":5603,"context_line":""},{"line_number":5604,"context_line":"        :param objs: a dict that maps object references to dicts that describe"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_3b7700a1","line":5601,"range":{"start_line":5601,"start_character":38,"end_line":5601,"end_character":42},"updated":"2016-09-08 10:36:43.000000000","message":"nits) s/frgs/frags/","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":5598,"context_line":"    def _setup_nodes_and_do_GET(self, objs, node_state):"},{"line_number":5599,"context_line":"        \"\"\""},{"line_number":5600,"context_line":"        A helper method that creates object fragments, stashes them in temp"},{"line_number":5601,"context_line":"        dirs, and then moves selected frgs back into the hash_dirs on each node"},{"line_number":5602,"context_line":"        according to a specified desired node state description."},{"line_number":5603,"context_line":""},{"line_number":5604,"context_line":"        :param objs: a dict that maps object references to dicts that describe"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_e434ee4b","line":5601,"range":{"start_line":5601,"start_character":38,"end_line":5601,"end_character":42},"in_reply_to":"9a89bdaa_3b7700a1","updated":"2016-09-13 18:49:29.000000000","message":"Done","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":5631,"context_line":"        # map nodes to hash dirs"},{"line_number":5632,"context_line":"        node_hash_dirs \u003d {}"},{"line_number":5633,"context_line":"        node_tmp_dirs \u003d collections.defaultdict(dict)"},{"line_number":5634,"context_line":"        for node_index, node in enumerate(nodes):"},{"line_number":5635,"context_line":"            node_hash_dirs[node_index] \u003d os.path.join("},{"line_number":5636,"context_line":"                _testdir, node[\u0027device\u0027], storage_directory("},{"line_number":5637,"context_line":"                    diskfile.get_data_dir(ec_policy),"},{"line_number":5638,"context_line":"                    partition, hash_path(\u0027a\u0027, container_name, obj_name)))"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_bb0f9035","line":5635,"range":{"start_line":5634,"start_character":8,"end_line":5635,"end_character":8},"updated":"2016-09-08 10:36:43.000000000","message":"Although it\u0027s okay, node already has the index info like node[\u0027index\u0027]?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":5631,"context_line":"        # map nodes to hash dirs"},{"line_number":5632,"context_line":"        node_hash_dirs \u003d {}"},{"line_number":5633,"context_line":"        node_tmp_dirs \u003d collections.defaultdict(dict)"},{"line_number":5634,"context_line":"        for node_index, node in enumerate(nodes):"},{"line_number":5635,"context_line":"            node_hash_dirs[node_index] \u003d os.path.join("},{"line_number":5636,"context_line":"                _testdir, node[\u0027device\u0027], storage_directory("},{"line_number":5637,"context_line":"                    diskfile.get_data_dir(ec_policy),"},{"line_number":5638,"context_line":"                    partition, hash_path(\u0027a\u0027, container_name, obj_name)))"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_64937efb","line":5635,"range":{"start_line":5634,"start_character":8,"end_line":5635,"end_character":8},"in_reply_to":"9a89bdaa_bb0f9035","updated":"2016-09-13 18:49:29.000000000","message":"Done. good call.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"2d8ff91f201612dfb7565e6d14f0dad10801b0cd","unresolved":false,"context_lines":[{"line_number":5815,"context_line":"        # back two responses with frag index 1, and will then return to node 0"},{"line_number":5816,"context_line":"        # for frag_index 0."},{"line_number":5817,"context_line":"        node_state \u003d {"},{"line_number":5818,"context_line":"            0: [dict(ref\u003d\u0027obj1\u0027, frag_index\u003d0, exts\u003d(\u0027.data\u0027,)),"},{"line_number":5819,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027,))],"},{"line_number":5820,"context_line":"            1: [dict(ref\u003d\u0027obj2\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))],"},{"line_number":5821,"context_line":"            2: [dict(ref\u003d\u0027obj3\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))]"},{"line_number":5822,"context_line":"        }"},{"line_number":5823,"context_line":""},{"line_number":5824,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":5825,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":5826,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"},{"line_number":5827,"context_line":""},{"line_number":5828,"context_line":"        # if all we have across nodes are frags with same index then expect a"},{"line_number":5829,"context_line":"        # 404 (the third, \u0027extra\u0027, obj server GET will return 404 because it"}],"source_content_type":"text/x-python","patch_set":39,"id":"9a89bdaa_f64dafc4","line":5826,"range":{"start_line":5818,"start_character":0,"end_line":5826,"end_character":57},"updated":"2016-09-08 10:36:43.000000000","message":"I\u0027m curious why we can fetch obj1 which doesn\u0027t have .durable file? Am I missing something?","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":4608,"name":"Kota Tsuyuzaki","email":"bloodeagle40234@gmail.com","username":"tsuyuzaki-kota"},"change_message_id":"bdb0e798acc4a38c95db23321e9a7f65feb53eb9","unresolved":false,"context_lines":[{"line_number":5815,"context_line":"        # back two responses with frag index 1, and will then return to node 0"},{"line_number":5816,"context_line":"        # for frag_index 0."},{"line_number":5817,"context_line":"        node_state \u003d {"},{"line_number":5818,"context_line":"            0: [dict(ref\u003d\u0027obj1\u0027, frag_index\u003d0, exts\u003d(\u0027.data\u0027,)),"},{"line_number":5819,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027,))],"},{"line_number":5820,"context_line":"            1: [dict(ref\u003d\u0027obj2\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))],"},{"line_number":5821,"context_line":"            2: [dict(ref\u003d\u0027obj3\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))]"},{"line_number":5822,"context_line":"        }"},{"line_number":5823,"context_line":""},{"line_number":5824,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":5825,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":5826,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"},{"line_number":5827,"context_line":""},{"line_number":5828,"context_line":"        # if all we have across nodes are frags with same index then expect a"},{"line_number":5829,"context_line":"        # 404 (the third, \u0027extra\u0027, obj server GET will return 404 because it"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_b3acd9ee","line":5826,"range":{"start_line":5818,"start_character":0,"end_line":5826,"end_character":57},"in_reply_to":"7a8ec9b2_67f1c0cd","updated":"2016-09-14 10:28:27.000000000","message":"Gotcha, all objects has same body (i.e. same etag). Ok, the new version looks good. Done.","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"e44725868fe57415e4c5e042a395266e20b80f70","unresolved":false,"context_lines":[{"line_number":5815,"context_line":"        # back two responses with frag index 1, and will then return to node 0"},{"line_number":5816,"context_line":"        # for frag_index 0."},{"line_number":5817,"context_line":"        node_state \u003d {"},{"line_number":5818,"context_line":"            0: [dict(ref\u003d\u0027obj1\u0027, frag_index\u003d0, exts\u003d(\u0027.data\u0027,)),"},{"line_number":5819,"context_line":"                dict(ref\u003d\u0027obj1\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027,))],"},{"line_number":5820,"context_line":"            1: [dict(ref\u003d\u0027obj2\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))],"},{"line_number":5821,"context_line":"            2: [dict(ref\u003d\u0027obj3\u0027, frag_index\u003d1, exts\u003d(\u0027.data\u0027, \u0027.durable\u0027))]"},{"line_number":5822,"context_line":"        }"},{"line_number":5823,"context_line":""},{"line_number":5824,"context_line":"        resp \u003d self._setup_nodes_and_do_GET(objs, node_state)"},{"line_number":5825,"context_line":"        self.assertEqual(resp.status_int, 200)"},{"line_number":5826,"context_line":"        self.assertEqual(resp.body, objs[\u0027obj1\u0027][\u0027body\u0027])"},{"line_number":5827,"context_line":""},{"line_number":5828,"context_line":"        # if all we have across nodes are frags with same index then expect a"},{"line_number":5829,"context_line":"        # 404 (the third, \u0027extra\u0027, obj server GET will return 404 because it"}],"source_content_type":"text/x-python","patch_set":39,"id":"7a8ec9b2_67f1c0cd","line":5826,"range":{"start_line":5818,"start_character":0,"end_line":5826,"end_character":57},"in_reply_to":"9a89bdaa_f64dafc4","updated":"2016-09-13 18:49:29.000000000","message":"all 3 objs are at same timestamp -they are simulating the same object having multiple duplicated fragments - see comment line 5805\n\nI will change var names to make it less confusing","commit_id":"014a36110c28bbbe58a63fb672723a6b651fb3ba"}]}