)]}'
{"/COMMIT_MSG":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":54,"context_line":"for the same object and frag-index, only the newer non-durable"},{"line_number":55,"context_line":"fragment will be synced and removed on the first reconstructor"},{"line_number":56,"context_line":"pass. The durable fragment will be synced and removed on the next"},{"line_number":57,"context_line":"reconstructor pass."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Change-Id: I1d47b865e0a621f35d323bbed472a6cfd2a5971b"},{"line_number":60,"context_line":"Closes-Bug: 1778002"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"6fe55ef0_acf0fdbe","line":57,"updated":"2021-01-12 06:03:32.000000000","message":"I went down a long rabbit hole wanting to make it so we could somehow pass both frags along in the one pass.\nI started modifying the protocol to add a `:MISSING_CHECK: EXTRA` where you could pass other files, to make it upgradable down the track.. but the further I got the more hacking up of diskfile I needed and changes to how we handle yeild_hashes.. so too much core changes and started getting messy, esp when we want to move to a tsync.\n\nLong story short, we are enabling passing no-durable frags, so the right thing to do is when it\u0027s enabled is to provide lastest ts_data. So I\u0027m now ok with having to pass the non durable in the first and the older durable in a second pass 😊","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":54,"context_line":"for the same object and frag-index, only the newer non-durable"},{"line_number":55,"context_line":"fragment will be synced and removed on the first reconstructor"},{"line_number":56,"context_line":"pass. The durable fragment will be synced and removed on the next"},{"line_number":57,"context_line":"reconstructor pass."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Change-Id: I1d47b865e0a621f35d323bbed472a6cfd2a5971b"},{"line_number":60,"context_line":"Closes-Bug: 1778002"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":2,"id":"cbeae313_ae6a31d5","line":57,"in_reply_to":"6fe55ef0_acf0fdbe","updated":"2021-01-12 15:23:42.000000000","message":"yeah, I also toyed with being able to send multiple timestamps per object per cycle (although I didn\u0027t think of a whole new handshake!) but decided to live with the compromise of one ts per cycle.\n\nThere\u0027s a trade-off: it\u0027s an uncommon case to have a stranded durable (we hope!) vs a lot of work to sync it in same cycle as the durable.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":13,"context_line":"    on a handoff"},{"line_number":14,"context_line":"  - add unit test to test_ssync.py"},{"line_number":15,"context_line":"  - add unit test for SsyncSender detecting not-upgraded receiver"},{"line_number":16,"context_line":"  - add diskfile unit test for legacy durable having \u0027durable\u0027 info"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Previously, ssync would not sync nor cleanup non-durable data"},{"line_number":19,"context_line":"fragments on handoffs. When the reconstructor is syncing objects from"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"0a434e6d_40832eee","line":16,"updated":"2021-01-12 23:04:59.000000000","message":"yeah yeah, more tests","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":44,"context_line":"  - The object server includes a new \u0027X-Backend-Accept-No-Commit\u0027"},{"line_number":45,"context_line":"    header in its response to SSYNC requests. This indicates to the"},{"line_number":46,"context_line":"    sender that the receiver has been upgraded to understand the"},{"line_number":47,"context_line":"    \u0027X-Backend-No-Commit\u0027 header."},{"line_number":48,"context_line":"  - The SsyncReceiver is enhanced to consider non-durable data when"},{"line_number":49,"context_line":"    determining if the sender\u0027s data is wanted or not."},{"line_number":50,"context_line":"  - The object server PUT method is enhanced to check for and"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"3f80e6da_e9665439","line":47,"updated":"2021-01-12 23:04:59.000000000","message":"this addresses the upgrade path; i\u0027m curious what the fallback path looks like","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":54,"context_line":"for the same object and frag-index, only the newer non-durable"},{"line_number":55,"context_line":"fragment will be synced and removed on the first reconstructor"},{"line_number":56,"context_line":"pass. The durable fragment will be synced and removed on the next"},{"line_number":57,"context_line":"reconstructor pass."},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"Change-Id: I1d47b865e0a621f35d323bbed472a6cfd2a5971b"},{"line_number":60,"context_line":"Closes-Bug: 1778002"}],"source_content_type":"text/x-gerrit-commit-message","patch_set":3,"id":"08fa744b_a3f90a42","line":57,"updated":"2021-01-12 23:04:59.000000000","message":"good caveat to keep in mind","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"swift/obj/diskfile.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":3483,"context_line":"            # sort the frag set into ascending frag_index order"},{"line_number":3484,"context_line":"            frag_set.sort(key\u003dlambda info: info[\u0027frag_index\u0027])"},{"line_number":3485,"context_line":"            frag \u003d frag_set[0]"},{"line_number":3486,"context_line":"            timestamp \u003d frag[\u0027timestamp\u0027]"},{"line_number":3487,"context_line":"            frag_sets[timestamp] \u003d frag_set"},{"line_number":3488,"context_line":"            for frag in frag_set:"},{"line_number":3489,"context_line":"                # a data file marked as durable may supersede a legacy durable"}],"source_content_type":"text/x-python","patch_set":2,"id":"6ae61f84_0dee2c1e","line":3486,"updated":"2021-01-12 06:03:32.000000000","message":"Not sure I see the point in splitting this over 2 lines as the `frag` variable it over written in the 2 fors later in the method.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":3483,"context_line":"            # sort the frag set into ascending frag_index order"},{"line_number":3484,"context_line":"            frag_set.sort(key\u003dlambda info: info[\u0027frag_index\u0027])"},{"line_number":3485,"context_line":"            frag \u003d frag_set[0]"},{"line_number":3486,"context_line":"            timestamp \u003d frag[\u0027timestamp\u0027]"},{"line_number":3487,"context_line":"            frag_sets[timestamp] \u003d frag_set"},{"line_number":3488,"context_line":"            for frag in frag_set:"},{"line_number":3489,"context_line":"                # a data file marked as durable may supersede a legacy durable"}],"source_content_type":"text/x-python","patch_set":2,"id":"716789a5_d2f0285a","line":3486,"in_reply_to":"6ae61f84_0dee2c1e","updated":"2021-01-12 15:23:42.000000000","message":"will fix\n\n(at one point I was referencing frag at line 3502 and pycharm whined about it maybe not being set, so I explicitly set it here. Then I added \u0027for frag in frag_set:\u0027 at line 3502, so I no longer need frag to be defined here.)","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":1631,"context_line":"                object_path \u003d os.path.join(suffix_path, object_hash)"},{"line_number":1632,"context_line":"                try:"},{"line_number":1633,"context_line":"                    results \u003d self.cleanup_ondisk_files("},{"line_number":1634,"context_line":"                        object_path, **kwargs)"},{"line_number":1635,"context_line":"                    if results[\u0027files\u0027]:"},{"line_number":1636,"context_line":"                        found_files \u003d True"},{"line_number":1637,"context_line":"                    timestamps \u003d {}"}],"source_content_type":"text/x-python","patch_set":3,"id":"ab5b4f3b_53f4eccc","line":1634,"updated":"2021-01-12 23:04:59.000000000","message":"so we\u0027ve *always* plumbed **kwargs through here","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":1635,"context_line":"                    if results[\u0027files\u0027]:"},{"line_number":1636,"context_line":"                        found_files \u003d True"},{"line_number":1637,"context_line":"                    timestamps \u003d {}"},{"line_number":1638,"context_line":"                    for ts_key, info_key, info_ts_key in key_preference:"},{"line_number":1639,"context_line":"                        if info_key not in results:"},{"line_number":1640,"context_line":"                            continue"},{"line_number":1641,"context_line":"                        info \u003d results[info_key]"}],"source_content_type":"text/x-python","patch_set":3,"id":"42130a05_dd369136","line":1638,"updated":"2021-01-12 23:04:59.000000000","message":"the ts_key is the key in the yeilded \"timestamps\" map\ninfo_key and info_ts_key are like sub-dict notation to dig out of \"results\" from get_ondisk_files\n\nts_key \u003d\u003e get_ondisk_files_results.info_key.info_ts_key","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":1635,"context_line":"                    if results[\u0027files\u0027]:"},{"line_number":1636,"context_line":"                        found_files \u003d True"},{"line_number":1637,"context_line":"                    timestamps \u003d {}"},{"line_number":1638,"context_line":"                    for ts_key, info_key, info_ts_key in key_preference:"},{"line_number":1639,"context_line":"                        if info_key not in results:"},{"line_number":1640,"context_line":"                            continue"},{"line_number":1641,"context_line":"                        info \u003d results[info_key]"}],"source_content_type":"text/x-python","patch_set":3,"id":"b5c5e64d_04d37de9","line":1638,"in_reply_to":"42130a05_dd369136","updated":"2021-01-13 15:32:57.000000000","message":"yes. I\u0027ll improve the comment at line 1615 and make the var names less specific to timestamps","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":1639,"context_line":"                        if info_key not in results:"},{"line_number":1640,"context_line":"                            continue"},{"line_number":1641,"context_line":"                        info \u003d results[info_key]"},{"line_number":1642,"context_line":"                        if info_ts_key in info:"},{"line_number":1643,"context_line":"                            timestamps[ts_key] \u003d info[info_ts_key]"},{"line_number":1644,"context_line":"                    if \u0027ts_data\u0027 not in timestamps:"},{"line_number":1645,"context_line":"                        # file sets that do not include a .data or .ts"}],"source_content_type":"text/x-python","patch_set":3,"id":"c4f268f4_1a69f4a6","line":1642,"updated":"2021-01-12 23:04:59.000000000","message":"realistically, this is for when \u0027durable\u0027 is not in result.data_info\n\n... which I think just means it\u0027s replicated","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":1639,"context_line":"                        if info_key not in results:"},{"line_number":1640,"context_line":"                            continue"},{"line_number":1641,"context_line":"                        info \u003d results[info_key]"},{"line_number":1642,"context_line":"                        if info_ts_key in info:"},{"line_number":1643,"context_line":"                            timestamps[ts_key] \u003d info[info_ts_key]"},{"line_number":1644,"context_line":"                    if \u0027ts_data\u0027 not in timestamps:"},{"line_number":1645,"context_line":"                        # file sets that do not include a .data or .ts"}],"source_content_type":"text/x-python","patch_set":3,"id":"1f8e15bf_1c4c1e74","line":1642,"in_reply_to":"c4f268f4_1a69f4a6","updated":"2021-01-13 15:32:57.000000000","message":"yes.\n\nalso, I meant to changes the var names now we\u0027re not just dealing with timestamps","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":3454,"context_line":""},{"line_number":3455,"context_line":"        When a frag_prefs arg is provided, including an empty list, there is no"},{"line_number":3456,"context_line":"        requirement for there to be a durable file at the same timestamp as a"},{"line_number":3457,"context_line":"        data file that is chosen to construct the disk file"},{"line_number":3458,"context_line":""},{"line_number":3459,"context_line":"        :param exts: dict of lists of file info, keyed by extension"},{"line_number":3460,"context_line":"        :param results: a dict that may be updated with results"}],"source_content_type":"text/x-python","patch_set":3,"id":"5a96e08a_a67683f6","line":3457,"updated":"2021-01-12 23:04:59.000000000","message":"It may not be the most *obvious* interface - but we have to give respect that it\u0027s *documented*","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":3499,"context_line":"                # is defined by a legacy .durable, so always mark all data"},{"line_number":3500,"context_line":"                # frags as durable here"},{"line_number":3501,"context_line":"                for frag in frag_set:"},{"line_number":3502,"context_line":"                    frag[\u0027durable\u0027] \u003d True"},{"line_number":3503,"context_line":"                break  # ignore frags that are older than durable timestamp"},{"line_number":3504,"context_line":""},{"line_number":3505,"context_line":"        # Choose which frag set to use"}],"source_content_type":"text/x-python","patch_set":3,"id":"d8a82a7e_ddaf052c","line":3502,"updated":"2021-01-12 23:04:59.000000000","message":"this sure makes it look like previously durable_frag_set would have had dicts with \"durable: False\"\n\n(Pdb) !durable_info\n{\u0027timestamp\u0027: 1610481235.00000_0000000000000000, \u0027ext\u0027: \u0027.durable\u0027, \u0027frag_index\u0027: None, \u0027filename\u0027: \u00271610481235.00000.durable\u0027, \u0027ctype_timestamp\u0027: None}\n(Pdb) !frag_set\n[{\u0027timestamp\u0027: 1610481235.00000_0000000000000000, \u0027durable\u0027: False, \u0027filename\u0027: \u00271610481235.00000#0.data\u0027, \u0027ext\u0027: \u0027.data\u0027, \u0027ctype_timestamp\u0027: None, \u0027frag_index\u0027: 0}]\n\nand I guess it was; but previously the \"incorrectness\" of the internal data-structures didn\u0027t leak out of any of the interfaces (we still *yielded* the durable legacy frags so no one cared about the key value)","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":3499,"context_line":"                # is defined by a legacy .durable, so always mark all data"},{"line_number":3500,"context_line":"                # frags as durable here"},{"line_number":3501,"context_line":"                for frag in frag_set:"},{"line_number":3502,"context_line":"                    frag[\u0027durable\u0027] \u003d True"},{"line_number":3503,"context_line":"                break  # ignore frags that are older than durable timestamp"},{"line_number":3504,"context_line":""},{"line_number":3505,"context_line":"        # Choose which frag set to use"}],"source_content_type":"text/x-python","patch_set":3,"id":"0a9b38cd_fddce34c","line":3502,"in_reply_to":"d8a82a7e_ddaf052c","updated":"2021-01-13 15:32:57.000000000","message":"Until this point the frag dict actually describes the data file properties, rather than the frag properties. The two are the same for a modern durable data file. By setting \u0027durable\u0027 for *legacy* data files here, we make the dict also describe the frag accurately in that legacy case.","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"swift/obj/reconstructor.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":892,"context_line":"                        node[\u0027index\u0027])"},{"line_number":893,"context_line":"                    success, in_sync_objs \u003d ssync_sender("},{"line_number":894,"context_line":"                        self, node, job, job[\u0027suffixes\u0027],"},{"line_number":895,"context_line":"                        include_non_durable\u003dTrue)()"},{"line_number":896,"context_line":"                    if success:"},{"line_number":897,"context_line":"                        syncd_with +\u003d 1"},{"line_number":898,"context_line":"                        reverted_objs.update(in_sync_objs)"}],"source_content_type":"text/x-python","patch_set":3,"id":"049177f0_fb65be22","line":895,"updated":"2021-01-12 23:04:59.000000000","message":"sync never includes durables and revert always does - i like that it\u0027s explicit","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":false,"context_lines":[{"line_number":892,"context_line":"                        node[\u0027index\u0027])"},{"line_number":893,"context_line":"                    success, in_sync_objs \u003d ssync_sender("},{"line_number":894,"context_line":"                        self, node, job, job[\u0027suffixes\u0027],"},{"line_number":895,"context_line":"                        include_non_durable\u003dTrue)()"},{"line_number":896,"context_line":"                    if success:"},{"line_number":897,"context_line":"                        syncd_with +\u003d 1"},{"line_number":898,"context_line":"                        reverted_objs.update(in_sync_objs)"}],"source_content_type":"text/x-python","patch_set":3,"id":"0862806f_10979573","line":895,"in_reply_to":"049177f0_fb65be22","updated":"2021-01-13 15:32:57.000000000","message":"Ack","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"swift/obj/server.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":1052,"context_line":"                    request.headers.get(\u0027X-Backend-No-Commit\u0027, False)):"},{"line_number":1053,"context_line":"                writer.commit(request.timestamp)"},{"line_number":1054,"context_line":"            if multi_stage_mime_state:"},{"line_number":1055,"context_line":"                self._drain_mime_request(**multi_stage_mime_state)"},{"line_number":1056,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":1057,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1058,"context_line":"        except ChunkReadError:"}],"source_content_type":"text/x-python","patch_set":3,"id":"75384b4b_dc911ace","line":1055,"updated":"2021-01-12 23:04:59.000000000","message":"so this looks like a multi_state_mime request could also still do a skip commit - but in practice I guess they\u0027re mutually exclusive.","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":1052,"context_line":"                    request.headers.get(\u0027X-Backend-No-Commit\u0027, False)):"},{"line_number":1053,"context_line":"                writer.commit(request.timestamp)"},{"line_number":1054,"context_line":"            if multi_stage_mime_state:"},{"line_number":1055,"context_line":"                self._drain_mime_request(**multi_stage_mime_state)"},{"line_number":1056,"context_line":"        except (DiskFileXattrNotSupported, DiskFileNoSpace):"},{"line_number":1057,"context_line":"            return HTTPInsufficientStorage(drive\u003ddevice, request\u003drequest)"},{"line_number":1058,"context_line":"        except ChunkReadError:"}],"source_content_type":"text/x-python","patch_set":3,"id":"1289b4c7_c733a274","line":1055,"in_reply_to":"75384b4b_dc911ace","updated":"2021-01-13 15:32:57.000000000","message":"yes. that *may* prove useful in a probe test??","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"swift/obj/ssync_receiver.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":58,"context_line":"            elif k \u003d\u003d \u0027t\u0027:"},{"line_number":59,"context_line":"                result[\u0027ts_ctype\u0027] \u003d Timestamp(t_data, delta\u003dint(v, 16))"},{"line_number":60,"context_line":"            elif k \u003d\u003d \u0027durable\u0027:"},{"line_number":61,"context_line":"                result[\u0027durable\u0027] \u003d utils.config_true_value(v)"},{"line_number":62,"context_line":"    return result"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"2ee72bb8_5e04f113","line":61,"updated":"2021-01-12 06:03:32.000000000","message":"Still not 100% with the durable (full word here rather then something like \u0027d\u0027) but I can live with it 😊","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":58,"context_line":"            elif k \u003d\u003d \u0027t\u0027:"},{"line_number":59,"context_line":"                result[\u0027ts_ctype\u0027] \u003d Timestamp(t_data, delta\u003dint(v, 16))"},{"line_number":60,"context_line":"            elif k \u003d\u003d \u0027durable\u0027:"},{"line_number":61,"context_line":"                result[\u0027durable\u0027] \u003d utils.config_true_value(v)"},{"line_number":62,"context_line":"    return result"},{"line_number":63,"context_line":""},{"line_number":64,"context_line":""}],"source_content_type":"text/x-python","patch_set":2,"id":"9feba807_26bcad83","line":61,"in_reply_to":"2ee72bb8_5e04f113","updated":"2021-01-12 15:23:42.000000000","message":"I\u0027d be happy with \u0027d\u0027 - let\u0027s use whatever most humans find most intuitive, the machines don\u0027t care","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":283,"context_line":"        except exceptions.DiskFileDeleted as err:"},{"line_number":284,"context_line":"            result \u003d {\u0027ts_data\u0027: err.timestamp}"},{"line_number":285,"context_line":"        except exceptions.DiskFileError:"},{"line_number":286,"context_line":"            # e.g. a non-durable EC frag"},{"line_number":287,"context_line":"            result \u003d {}"},{"line_number":288,"context_line":"        else:"},{"line_number":289,"context_line":"            result \u003d {"}],"source_content_type":"text/x-python","patch_set":2,"id":"9e78ffc8_2eb2352f","line":286,"updated":"2021-01-12 06:03:32.000000000","message":"I guess it would be nice if we relied on something other then a DiskFileError for the case of a non durable ec frag now that we are supporting it.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":283,"context_line":"        except exceptions.DiskFileDeleted as err:"},{"line_number":284,"context_line":"            result \u003d {\u0027ts_data\u0027: err.timestamp}"},{"line_number":285,"context_line":"        except exceptions.DiskFileError:"},{"line_number":286,"context_line":"            # e.g. a non-durable EC frag"},{"line_number":287,"context_line":"            result \u003d {}"},{"line_number":288,"context_line":"        else:"},{"line_number":289,"context_line":"            result \u003d {"}],"source_content_type":"text/x-python","patch_set":2,"id":"f48d5c4c_e42e7658","line":286,"in_reply_to":"9e78ffc8_2eb2352f","updated":"2021-01-12 15:23:42.000000000","message":"Interesting point. The Diskfile interface has become a bit messy - the constructor args define which on disk files will be used for the open(), rather rather open taking those args e.g. open(ts, frag_index)\n\nWe probably could pass frag_prefs\u003d[] get_diskfile_from_hash() above and then query the quite detailed diskfile info to figure out if we have a durable, similar to what we do below to find any non-durable.\n\nI guess I was trying to minimise the changes.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":295,"context_line":"             df.durable_timestamp \u003c remote[\u0027ts_data\u0027]) and"},{"line_number":296,"context_line":"                df.fragments and"},{"line_number":297,"context_line":"                remote[\u0027ts_data\u0027] in df.fragments and"},{"line_number":298,"context_line":"                self.frag_index in df.fragments[remote[\u0027ts_data\u0027]]):"},{"line_number":299,"context_line":"            # The remote is offering a fragment that we already have but is"},{"line_number":300,"context_line":"            # *newer* than anything durable that we have"},{"line_number":301,"context_line":"            if remote[\u0027durable\u0027] and make_durable:"}],"source_content_type":"text/x-python","patch_set":2,"id":"a1bc0976_d61846fe","line":298,"updated":"2021-01-12 06:03:32.000000000","message":"phew that if statement gave me some headaches, read it wrong for a while and was wondering how are we making things durable when we can\u0027t tell.. but turns out, no just a crazy complicated if statement :P","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":295,"context_line":"             df.durable_timestamp \u003c remote[\u0027ts_data\u0027]) and"},{"line_number":296,"context_line":"                df.fragments and"},{"line_number":297,"context_line":"                remote[\u0027ts_data\u0027] in df.fragments and"},{"line_number":298,"context_line":"                self.frag_index in df.fragments[remote[\u0027ts_data\u0027]]):"},{"line_number":299,"context_line":"            # The remote is offering a fragment that we already have but is"},{"line_number":300,"context_line":"            # *newer* than anything durable that we have"},{"line_number":301,"context_line":"            if remote[\u0027durable\u0027] and make_durable:"}],"source_content_type":"text/x-python","patch_set":2,"id":"fdac6066_66b5dae6","line":298,"in_reply_to":"a1bc0976_d61846fe","updated":"2021-01-12 15:23:42.000000000","message":"writing it gave me headaches :)","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":275,"context_line":"        try:"},{"line_number":276,"context_line":"            df \u003d self.diskfile_mgr.get_diskfile_from_hash("},{"line_number":277,"context_line":"                self.device, self.partition, remote[\u0027object_hash\u0027],"},{"line_number":278,"context_line":"                self.policy, frag_index\u003dself.frag_index, open_expired\u003dTrue)"},{"line_number":279,"context_line":"        except exceptions.DiskFileNotExist:"},{"line_number":280,"context_line":"            return {}"},{"line_number":281,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"78227c8c_b215c358","line":278,"updated":"2021-01-12 23:04:59.000000000","message":"I wonder why we don\u0027t have to say frag_prefs\u003d[] to get an non-durable here?","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":275,"context_line":"        try:"},{"line_number":276,"context_line":"            df \u003d self.diskfile_mgr.get_diskfile_from_hash("},{"line_number":277,"context_line":"                self.device, self.partition, remote[\u0027object_hash\u0027],"},{"line_number":278,"context_line":"                self.policy, frag_index\u003dself.frag_index, open_expired\u003dTrue)"},{"line_number":279,"context_line":"        except exceptions.DiskFileNotExist:"},{"line_number":280,"context_line":"            return {}"},{"line_number":281,"context_line":"        try:"}],"source_content_type":"text/x-python","patch_set":3,"id":"bb030bd9_eae1a04d","line":278,"in_reply_to":"78227c8c_b215c358","updated":"2021-01-13 15:32:57.000000000","message":"see reply to Matt on previous patch. in short, we could probably re-write more of this method if motivated.\n\nhttps://review.opendev.org/c/openstack/swift/+/770047/2/swift/obj/ssync_receiver.py#286\n\nalso https://review.opendev.org/c/openstack/swift/+/770047/2/swift/obj/ssync_sender.py#363","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            if not remote[\u0027durable\u0027]:"},{"line_number":318,"context_line":"                # We have the frag so make sure our ts_data will prevent it"},{"line_number":319,"context_line":"                # being wanted even if our copy is non-durable"},{"line_number":320,"context_line":"                result[\u0027ts_data\u0027] \u003d remote[\u0027ts_data\u0027]"},{"line_number":321,"context_line":"        return result"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"    def _check_missing(self, line):"}],"source_content_type":"text/x-python","patch_set":3,"id":"db53dcde_940c3df4","line":320,"updated":"2021-01-12 23:04:59.000000000","message":"nit: something about \"if durable, if not durable\" in series bugs me\n\nI might extract the \"mark local matching non-druable as durable\" to a helper so it\u0027s not so indented\n\nthe new \"if not remote.durable\" isn\u0027t obvious, i\u0027ll need to find the test","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":317,"context_line":"            if not remote[\u0027durable\u0027]:"},{"line_number":318,"context_line":"                # We have the frag so make sure our ts_data will prevent it"},{"line_number":319,"context_line":"                # being wanted even if our copy is non-durable"},{"line_number":320,"context_line":"                result[\u0027ts_data\u0027] \u003d remote[\u0027ts_data\u0027]"},{"line_number":321,"context_line":"        return result"},{"line_number":322,"context_line":""},{"line_number":323,"context_line":"    def _check_missing(self, line):"}],"source_content_type":"text/x-python","patch_set":3,"id":"98fcacc4_9ad2538c","line":320,"in_reply_to":"db53dcde_940c3df4","updated":"2021-01-13 15:32:57.000000000","message":"fair point. IIRC I was bothered by indent depth, but its not too bad","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"swift/obj/ssync_sender.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":286,"context_line":"            connection.send(b\u0027%x\\r\\n%s\\r\\n\u0027 % (len(msg), msg))"},{"line_number":287,"context_line":"        # an empty frag_prefs list is sufficient to get non-durable frags"},{"line_number":288,"context_line":"        # yielded, in which case an older durable frag will not be yielded"},{"line_number":289,"context_line":"        frag_prefs \u003d [] if self.include_non_durable else None"},{"line_number":290,"context_line":"        hash_gen \u003d self.df_mgr.yield_hashes("},{"line_number":291,"context_line":"            self.job[\u0027device\u0027], self.job[\u0027partition\u0027],"},{"line_number":292,"context_line":"            self.job[\u0027policy\u0027], self.suffixes,"}],"source_content_type":"text/x-python","patch_set":2,"id":"56fefdeb_92b74e42","line":289,"updated":"2021-01-12 06:03:32.000000000","message":"Nice trick. Passing an empty frag_prefs will default to getting the latest frag durable or not.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":360,"context_line":"                df \u003d self.df_mgr.get_diskfile_from_hash("},{"line_number":361,"context_line":"                    self.job[\u0027device\u0027], self.job[\u0027partition\u0027], object_hash,"},{"line_number":362,"context_line":"                    self.job[\u0027policy\u0027], frag_index\u003dself.job.get(\u0027frag_index\u0027),"},{"line_number":363,"context_line":"                    open_expired\u003dTrue, frag_prefs\u003dfrag_prefs)"},{"line_number":364,"context_line":"            except exceptions.DiskFileNotExist:"},{"line_number":365,"context_line":"                continue"},{"line_number":366,"context_line":"            url_path \u003d urllib.parse.quote("}],"source_content_type":"text/x-python","patch_set":2,"id":"80944f00_a203b3ad","line":363,"updated":"2021-01-12 06:03:32.000000000","message":"Hrm, get_diskfile_from_hash doesn\u0027t seem to be passing frag_prefs thorugh to cleanup_ondisk_files which is needed as it\u0027s the link down to the ECDiskFileManager\u0027s _process_ondisk_files, which is were frag_prefs are used.\n\nSo wouldn\u0027t this only ever give back the latest durable frag?\n\nWe probably need something like:\n\n diff --git a/swift/obj/diskfile.py b/swift/obj/diskfile.py\n index 5710a09c9..3a333b4fa 100644\n --- a/swift/obj/diskfile.py\n +++ b/swift/obj/diskfile.py\n @@ -1478,7 +1478,7 @@ class BaseDiskFileManager(object):\n              dev_path, get_data_dir(policy), str(partition), object_hash[-3:],\n              object_hash)\n          try:\n -            filenames \u003d self.cleanup_ondisk_files(object_path)[\u0027files\u0027]\n +            filenames \u003d self.cleanup_ondisk_files(object_path, **kwargs)[\u0027files\u0027]\n          except OSError as err:\n              if err.errno \u003d\u003d errno.ENOTDIR:\n                  # The made-up filename is so that the eventual dirpath()","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":360,"context_line":"                df \u003d self.df_mgr.get_diskfile_from_hash("},{"line_number":361,"context_line":"                    self.job[\u0027device\u0027], self.job[\u0027partition\u0027], object_hash,"},{"line_number":362,"context_line":"                    self.job[\u0027policy\u0027], frag_index\u003dself.job.get(\u0027frag_index\u0027),"},{"line_number":363,"context_line":"                    open_expired\u003dTrue, frag_prefs\u003dfrag_prefs)"},{"line_number":364,"context_line":"            except exceptions.DiskFileNotExist:"},{"line_number":365,"context_line":"                continue"},{"line_number":366,"context_line":"            url_path \u003d urllib.parse.quote("}],"source_content_type":"text/x-python","patch_set":2,"id":"0f89d4d0_9e254991","line":363,"in_reply_to":"80944f00_a203b3ad","updated":"2021-01-12 15:23:42.000000000","message":"I think it\u0027s OK. In get_diskfile_from_hash() the cleanup_ondisk_files is just being used to cleanup; at the end of the method the diskfile_cls is constructed with the **kwargs so frag_prefs makes it through.\n\nThere does seem to be some redundancy in that we os.listdir in get_diskfile_from_hash() and then os.listdir again when we call df.open() below.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":268,"context_line":"        hash_gen \u003d self.df_mgr.yield_hashes("},{"line_number":269,"context_line":"            self.job[\u0027device\u0027], self.job[\u0027partition\u0027],"},{"line_number":270,"context_line":"            self.job[\u0027policy\u0027], self.suffixes,"},{"line_number":271,"context_line":"            frag_index\u003dself.job.get(\u0027frag_index\u0027))"},{"line_number":272,"context_line":"        if self.remote_check_objs is not None:"},{"line_number":273,"context_line":"            hash_gen \u003d six.moves.filter("},{"line_number":274,"context_line":"                lambda objhash_timestamps:"}],"source_content_type":"text/x-python","patch_set":3,"id":"ddfa3279_14fb7d0d","side":"PARENT","line":271,"updated":"2021-01-12 23:04:59.000000000","message":"so the **kwargs plumbing was for existing use of frag_index","commit_id":"128f1995088d11e6d41902271b683750fb2e770e"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":46,"context_line":"            extra_parts.append(\u0027t:%x\u0027 % delta)"},{"line_number":47,"context_line":"    if \u0027durable\u0027 in kwargs and kwargs[\u0027durable\u0027] is False:"},{"line_number":48,"context_line":"        # only send durable in the less common case that it is False"},{"line_number":49,"context_line":"        extra_parts.append(\u0027durable:%s\u0027 % kwargs[\u0027durable\u0027])"},{"line_number":50,"context_line":"    if extra_parts:"},{"line_number":51,"context_line":"        msg \u003d \u0027%s %s\u0027 % (msg, \u0027,\u0027.join(extra_parts))"},{"line_number":52,"context_line":"    return msg.encode(\u0027ascii\u0027)"}],"source_content_type":"text/x-python","patch_set":3,"id":"8ed89c34_c6fdecfe","line":49,"updated":"2021-01-12 23:04:59.000000000","message":"right i saw that in tests - unless it\u0027s durable\u003dFalse we assume it\u0027s durable","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":260,"context_line":"                    \u0027Expected status %s; got %s (%s)\u0027 %"},{"line_number":261,"context_line":"                    (http.HTTP_OK, response.status, err_msg))"},{"line_number":262,"context_line":"            if self.include_non_durable and not config_true_value("},{"line_number":263,"context_line":"                    response.getheader(\u0027x-backend-accept-no-commit\u0027, False)):"},{"line_number":264,"context_line":"                # fall back to legacy behaviour if receiver does not understand"},{"line_number":265,"context_line":"                # X-Backend-Commit"},{"line_number":266,"context_line":"                self.daemon.logger.warning("}],"source_content_type":"text/x-python","patch_set":3,"id":"8437ad64_e7cd7aba","line":263,"updated":"2021-01-12 23:04:59.000000000","message":"and for sync jobs we just don\u0027t care.  there\u0027s no check of the header.  we don\u0027t log any warning.","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":260,"context_line":"                    \u0027Expected status %s; got %s (%s)\u0027 %"},{"line_number":261,"context_line":"                    (http.HTTP_OK, response.status, err_msg))"},{"line_number":262,"context_line":"            if self.include_non_durable and not config_true_value("},{"line_number":263,"context_line":"                    response.getheader(\u0027x-backend-accept-no-commit\u0027, False)):"},{"line_number":264,"context_line":"                # fall back to legacy behaviour if receiver does not understand"},{"line_number":265,"context_line":"                # X-Backend-Commit"},{"line_number":266,"context_line":"                self.daemon.logger.warning("}],"source_content_type":"text/x-python","patch_set":3,"id":"03318833_9826c0e2","line":263,"in_reply_to":"8437ad64_e7cd7aba","updated":"2021-01-13 15:32:57.000000000","message":"right. I figured that we\u0027d only warn when actually prevented from taking action.\n\nThese warnings should be transient (during upgrade), so we may not be seen at all, and would be a shame to be distracted by harmless warnings from sync jobs.","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":266,"context_line":"                self.daemon.logger.warning("},{"line_number":267,"context_line":"                    \u0027ssync receiver %s does not accept non-durable fragments\u0027 %"},{"line_number":268,"context_line":"                    node_addr)"},{"line_number":269,"context_line":"                self.include_non_durable \u003d False"},{"line_number":270,"context_line":"        return connection, response"},{"line_number":271,"context_line":""},{"line_number":272,"context_line":"    def missing_check(self, connection, response):"}],"source_content_type":"text/x-python","patch_set":3,"id":"3586f72d_6eef6f02","line":269,"updated":"2021-01-12 23:04:59.000000000","message":"and obviously connect is handled before we start yielding hashes so it\u0027s perfectly reasonable to downgrade to legacy behavior here despite","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":285,"context_line":"            msg \u003d b\u0027:MISSING_CHECK: START\\r\\n\u0027"},{"line_number":286,"context_line":"            connection.send(b\u0027%x\\r\\n%s\\r\\n\u0027 % (len(msg), msg))"},{"line_number":287,"context_line":"        # an empty frag_prefs list is sufficient to get non-durable frags"},{"line_number":288,"context_line":"        # yielded, in which case an older durable frag will not be yielded"},{"line_number":289,"context_line":"        frag_prefs \u003d [] if self.include_non_durable else None"},{"line_number":290,"context_line":"        hash_gen \u003d self.df_mgr.yield_hashes("},{"line_number":291,"context_line":"            self.job[\u0027device\u0027], self.job[\u0027partition\u0027],"}],"source_content_type":"text/x-python","patch_set":3,"id":"115d3a96_3f90714f","line":288,"updated":"2021-01-12 23:04:59.000000000","message":"lawl?","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":360,"context_line":"                df \u003d self.df_mgr.get_diskfile_from_hash("},{"line_number":361,"context_line":"                    self.job[\u0027device\u0027], self.job[\u0027partition\u0027], object_hash,"},{"line_number":362,"context_line":"                    self.job[\u0027policy\u0027], frag_index\u003dself.job.get(\u0027frag_index\u0027),"},{"line_number":363,"context_line":"                    open_expired\u003dTrue, frag_prefs\u003dfrag_prefs)"},{"line_number":364,"context_line":"            except exceptions.DiskFileNotExist:"},{"line_number":365,"context_line":"                continue"},{"line_number":366,"context_line":"            url_path \u003d urllib.parse.quote("}],"source_content_type":"text/x-python","patch_set":3,"id":"ad25776e_0d48afe3","line":363,"updated":"2021-01-12 23:04:59.000000000","message":"same trick for opening non-durable and the \"accept-no-commit\" flag in connect would flop us into \"durable-only\" so this should be consistent with missing_check\n\nI don\u0027t think we ever reuse instances of Sender under concurrency","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":368,"context_line":"            try:"},{"line_number":369,"context_line":"                df.open()"},{"line_number":370,"context_line":"                if want.get(\u0027data\u0027):"},{"line_number":371,"context_line":"                    is_durable \u003d (df.durable_timestamp \u003d\u003d df.data_timestamp)"},{"line_number":372,"context_line":"                    # EC reconstructor may have passed a callback to build an"},{"line_number":373,"context_line":"                    # alternative diskfile - construct it using the metadata"},{"line_number":374,"context_line":"                    # from the data file only."}],"source_content_type":"text/x-python","patch_set":3,"id":"892bf725_c81be308","line":371,"updated":"2021-01-12 23:04:59.000000000","message":"i didn\u0027t see a property for this in diskfile.py - i\u0027m assuming this is the canonical way to spell this?","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":368,"context_line":"            try:"},{"line_number":369,"context_line":"                df.open()"},{"line_number":370,"context_line":"                if want.get(\u0027data\u0027):"},{"line_number":371,"context_line":"                    is_durable \u003d (df.durable_timestamp \u003d\u003d df.data_timestamp)"},{"line_number":372,"context_line":"                    # EC reconstructor may have passed a callback to build an"},{"line_number":373,"context_line":"                    # alternative diskfile - construct it using the metadata"},{"line_number":374,"context_line":"                    # from the data file only."}],"source_content_type":"text/x-python","patch_set":3,"id":"a76b9ae0_9d4ea85b","line":371,"in_reply_to":"892bf725_c81be308","updated":"2021-01-13 15:32:57.000000000","message":"I wondered about adding a property, and I am torn.\n\nDiskfile has become a complex beast. On the one hand it represents a fragment+metadata at a particular timestamp/frag_index. On the other hand it offers the wider view of everything that is on disk for an object (since we needed to parse all that to construct the Diskfile). So it can be confusing as to whether a Diskfile method is on the chosen_frag, or on the whole e.g. is_durable() - meaning the chosen_frag, or meaning there is durable data on disk?\n\nTurns out this equality is tested only here and a couple of times in tests.","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"test/probe/test_reconstructor_revert.py":[{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":367,"context_line":"        self.revive_drive(p_dev1)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"        # fire up reconstructor on handoff node only"},{"line_number":370,"context_line":"        hnode_id \u003d (hnodes[0][\u0027port\u0027] % 100) // 10"},{"line_number":371,"context_line":"        self.reconstructor.once(number\u003dhnode_id)"},{"line_number":372,"context_line":""},{"line_number":373,"context_line":"        # primary now has *non-durable* frag"}],"source_content_type":"text/x-python","patch_set":2,"id":"4e89c65b_406bb44a","line":370,"updated":"2021-01-12 06:03:32.000000000","message":"Lol, I guess that works, took me a while to figure out it\u0027s just getting the 3rd digit in the port.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":367,"context_line":"        self.revive_drive(p_dev1)"},{"line_number":368,"context_line":""},{"line_number":369,"context_line":"        # fire up reconstructor on handoff node only"},{"line_number":370,"context_line":"        hnode_id \u003d (hnodes[0][\u0027port\u0027] % 100) // 10"},{"line_number":371,"context_line":"        self.reconstructor.once(number\u003dhnode_id)"},{"line_number":372,"context_line":""},{"line_number":373,"context_line":"        # primary now has *non-durable* frag"}],"source_content_type":"text/x-python","patch_set":2,"id":"95c160c7_5fcbeac5","line":370,"in_reply_to":"4e89c65b_406bb44a","updated":"2021-01-12 15:23:42.000000000","message":"TBH I cut and paste that :(","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7233,"name":"Matthew Oliver","email":"matt@oliver.net.au","username":"mattoliverau"},"change_message_id":"431fc588f5dbaba5ddb20f3de0e6418f6ee8f104","unresolved":true,"context_lines":[{"line_number":384,"context_line":""},{"line_number":385,"context_line":"        # first primary now has *durable* frag - it *was* useful to sync the"},{"line_number":386,"context_line":"        # non-durable!"},{"line_number":387,"context_line":"        self.assert_direct_get_succeeds(onodes[0], opart)"},{"line_number":388,"context_line":""},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"if __name__ \u003d\u003d \"__main__\":"}],"source_content_type":"text/x-python","patch_set":2,"id":"c8038d20_0f4fc34b","line":387,"updated":"2021-01-12 06:03:32.000000000","message":"Can we confirm that it actaully make it durable via a revert/makedurable and not via a rebuild the frag?\n\nI mean I know it did as that\u0027s what the code is suppose to do. But it would be nice if we could show that in the test.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"8895de0988954a3e6ed9ecbcbd0b8fbce6cc4f34","unresolved":true,"context_lines":[{"line_number":384,"context_line":""},{"line_number":385,"context_line":"        # first primary now has *durable* frag - it *was* useful to sync the"},{"line_number":386,"context_line":"        # non-durable!"},{"line_number":387,"context_line":"        self.assert_direct_get_succeeds(onodes[0], opart)"},{"line_number":388,"context_line":""},{"line_number":389,"context_line":""},{"line_number":390,"context_line":"if __name__ \u003d\u003d \"__main__\":"}],"source_content_type":"text/x-python","patch_set":2,"id":"6ee998b7_c6ca5e8f","line":387,"in_reply_to":"c8038d20_0f4fc34b","updated":"2021-01-12 15:23:42.000000000","message":"Yes, I was thinking about that, and maybe I can do that by disabling the other primaries so that it would be impossible to reconstruct the frag.","commit_id":"1cb851979dcb59ac861ea77832931bbf87cd0972"}],"test/unit/obj/test_diskfile.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":2792,"context_line":"            \u00279373a92d072897b136b3fc06595b0456\u0027: {\u0027ts_data\u0027: old_ts,"},{"line_number":2793,"context_line":"                                                 \u0027durable\u0027: True},"},{"line_number":2794,"context_line":"            \u00279373a92d072897b136b3fc06595b7456\u0027: {\u0027ts_data\u0027: fresher_ts,"},{"line_number":2795,"context_line":"                                                 \u0027durable\u0027: True},"},{"line_number":2796,"context_line":"        }"},{"line_number":2797,"context_line":"        self._check_yield_hashes(POLICIES.default, suffix_map, expected,"},{"line_number":2798,"context_line":"                                 frag_index\u003d2)"}],"source_content_type":"text/x-python","patch_set":3,"id":"bdf1d443_2b8928b8","line":2795,"updated":"2021-01-12 23:04:59.000000000","message":"ah, this covers the fixup legacy nondurable fixup","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":3038,"context_line":"            \u00279373a92d072897b136b3fc06595b0456\u0027: {\u0027ts_data\u0027: ts1,"},{"line_number":3039,"context_line":"                                                 \u0027durable\u0027: True},"},{"line_number":3040,"context_line":"            \u00279373a92d072897b136b3fc06595b7456\u0027: {\u0027ts_data\u0027: ts2,"},{"line_number":3041,"context_line":"                                                 \u0027durable\u0027: False},"},{"line_number":3042,"context_line":"        }"},{"line_number":3043,"context_line":"        self._check_yield_hashes(POLICIES.default, suffix_map, expected,"},{"line_number":3044,"context_line":"                                 frag_index\u003d2, frag_prefs\u003d[])"}],"source_content_type":"text/x-python","patch_set":3,"id":"6d2144f8_ae838047","line":3041,"updated":"2021-01-12 23:04:59.000000000","message":"sketch - who\u0027s setting False?","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":3038,"context_line":"            \u00279373a92d072897b136b3fc06595b0456\u0027: {\u0027ts_data\u0027: ts1,"},{"line_number":3039,"context_line":"                                                 \u0027durable\u0027: True},"},{"line_number":3040,"context_line":"            \u00279373a92d072897b136b3fc06595b7456\u0027: {\u0027ts_data\u0027: ts2,"},{"line_number":3041,"context_line":"                                                 \u0027durable\u0027: False},"},{"line_number":3042,"context_line":"        }"},{"line_number":3043,"context_line":"        self._check_yield_hashes(POLICIES.default, suffix_map, expected,"},{"line_number":3044,"context_line":"                                 frag_index\u003d2, frag_prefs\u003d[])"}],"source_content_type":"text/x-python","patch_set":3,"id":"e67a1543_960acb8d","line":3041,"in_reply_to":"6d2144f8_ae838047","updated":"2021-01-13 15:32:57.000000000","message":"swift.obj.diskfile.ECDiskFileManager.parse_on_disk_filename\n\nthis is an EC specific test, so durable is always in the diskfile info dict","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"test/unit/obj/test_ssync_receiver.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":2593,"context_line":"        check_durable(\u0027yes\u0027)"},{"line_number":2594,"context_line":"        check_durable(\u0027true\u0027)"},{"line_number":2595,"context_line":"        check_durable(\u0027True\u0027)"},{"line_number":2596,"context_line":""},{"line_number":2597,"context_line":"    def test_encode_wanted(self):"},{"line_number":2598,"context_line":"        ts_iter \u003d make_timestamp_iter()"},{"line_number":2599,"context_line":"        old_t_data \u003d next(ts_iter)"}],"source_content_type":"text/x-python","patch_set":3,"id":"fb401d92_b68af03f","line":2596,"updated":"2021-01-12 23:04:59.000000000","message":"these tests give a good sense of how this parser works, quite nice","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"c5524fac78df5de7225a1e341851d8cf17445dee","unresolved":true,"context_lines":[{"line_number":2663,"context_line":"        }"},{"line_number":2664,"context_line":"        expected \u003d \u0027theremotehash d\u0027"},{"line_number":2665,"context_line":"        self.assertEqual(ssync_receiver.encode_wanted(remote, local),"},{"line_number":2666,"context_line":"                         expected)"},{"line_number":2667,"context_line":""},{"line_number":2668,"context_line":""},{"line_number":2669,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":3,"id":"8e48d657_2f6b2c07","line":2666,"updated":"2021-01-12 23:04:59.000000000","message":"not obvious to me yet why encoded_wanted didn\u0027t have to become durable aware.\n\n... I guess if we want a timestamp we want it - durable or not","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"},{"author":{"_account_id":7847,"name":"Alistair Coles","email":"alistairncoles@gmail.com","username":"acoles"},"change_message_id":"68d30d542b416af4014cbdc616ffc7201ecd887c","unresolved":true,"context_lines":[{"line_number":2663,"context_line":"        }"},{"line_number":2664,"context_line":"        expected \u003d \u0027theremotehash d\u0027"},{"line_number":2665,"context_line":"        self.assertEqual(ssync_receiver.encode_wanted(remote, local),"},{"line_number":2666,"context_line":"                         expected)"},{"line_number":2667,"context_line":""},{"line_number":2668,"context_line":""},{"line_number":2669,"context_line":"if __name__ \u003d\u003d \u0027__main__\u0027:"}],"source_content_type":"text/x-python","patch_set":3,"id":"ce64da3f_f4ff75d9","line":2666,"in_reply_to":"8e48d657_2f6b2c07","updated":"2021-01-13 15:32:57.000000000","message":"yes, sender sends only one timestamp per object per cycle so receiver either wants it or not, no need for receiver to indicate that \u0027durable at t1 is not wanted but non-durable at t2 is wanted\u0027","commit_id":"a841e1b1c33f871aa3ac78a1357bbf54b36dbbd5"}],"test/unit/obj/test_ssync_sender.py":[{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3196323ce4f5ddd0de760c3770844a03dcc00db8","unresolved":true,"context_lines":[{"line_number":425,"context_line":"        sender \u003d self._do_test_connect_include_non_durable(True, {})"},{"line_number":426,"context_line":"        # no \u0027x-backend-accept-no-commit\u0027 in response,"},{"line_number":427,"context_line":"        # sender.include_non_durable has been overridden"},{"line_number":428,"context_line":"        self.assertFalse(sender.include_non_durable)"},{"line_number":429,"context_line":"        warnings \u003d self.daemon_logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":430,"context_line":"        self.assertEqual([\u0027ssync receiver 1.2.3.4:5678 does not accept \u0027"},{"line_number":431,"context_line":"                          \u0027non-durable fragments\u0027], warnings)"}],"source_content_type":"text/x-python","patch_set":5,"id":"91b07485_91c8b4a3","line":428,"updated":"2021-01-21 19:25:29.000000000","message":"connecting to legacy server juts doesn\u0027t yield non-durables; easy peasy","commit_id":"620c5caee78de8b559753afa52c0e67f1efb4986"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3196323ce4f5ddd0de760c3770844a03dcc00db8","unresolved":true,"context_lines":[{"line_number":435,"context_line":"        sender \u003d self._do_test_connect_include_non_durable(True, resp_hdrs)"},{"line_number":436,"context_line":"        self.assertTrue(sender.include_non_durable)"},{"line_number":437,"context_line":"        warnings \u003d self.daemon_logger.get_lines_for_level(\u0027warning\u0027)"},{"line_number":438,"context_line":"        self.assertEqual([], warnings)"},{"line_number":439,"context_line":""},{"line_number":440,"context_line":"    def test_call(self):"},{"line_number":441,"context_line":"        def patch_sender(sender, available_map, send_map):"}],"source_content_type":"text/x-python","patch_set":5,"id":"15cda4f4_3ddc5b48","line":438,"updated":"2021-01-21 19:25:29.000000000","message":"these are pretty great","commit_id":"620c5caee78de8b559753afa52c0e67f1efb4986"},{"author":{"_account_id":1179,"name":"Clay Gerrard","email":"clay.gerrard@gmail.com","username":"clay-gerrard"},"change_message_id":"3196323ce4f5ddd0de760c3770844a03dcc00db8","unresolved":true,"context_lines":[{"line_number":1557,"context_line":"            \u0027ETag: %(ETag)s\\r\\n\u0027"},{"line_number":1558,"context_line":"            \u0027Some-Other-Header: value\\r\\n\u0027"},{"line_number":1559,"context_line":"            \u0027Unicode-Meta-Name: %(meta)s\\r\\n\u0027"},{"line_number":1560,"context_line":"            \u0027%(no_commit)s\u0027"},{"line_number":1561,"context_line":"            \u0027X-Timestamp: %(X-Timestamp)s\\r\\n\u0027"},{"line_number":1562,"context_line":"            \u0027\\r\\n\u0027"},{"line_number":1563,"context_line":"            \u0027\\r\\n\u0027"}],"source_content_type":"text/x-python","patch_set":5,"id":"b9818b46_907ba7e0","line":1560,"updated":"2021-01-21 19:25:29.000000000","message":"right, if not durable then send no-commit 👍","commit_id":"620c5caee78de8b559753afa52c0e67f1efb4986"}]}