)]}'
{"/PATCHSET_LEVEL":[{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"0078337d_36c54cf3","updated":"2021-11-02 16:46:21.000000000","message":"Some comments in-line and I think the other comments also need to be addressed.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":12670,"name":"Helen Walsh","email":"helen.walsh@emc.com","username":"walshh2"},"change_message_id":"257464edc9480407d04d17b990b815dc98e00766","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":1,"id":"cfbecf6f_2dc70ca3","updated":"2021-11-02 11:26:27.000000000","message":"nits only, a few suggestions for clarity","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"b11ecae0a8d2b26507da4bd16b7702e167c19986","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"8f9fd7bc_1f736982","updated":"2021-11-05 12:59:59.000000000","message":"Fix for the docs job noted inline.","commit_id":"7459a1bd5fbf302d95a09c2ef0114191af072904"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":2,"id":"4764c331_aeb7b971","updated":"2021-11-03 08:45:52.000000000","message":"Thanks for all the reviews!","commit_id":"7459a1bd5fbf302d95a09c2ef0114191af072904"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"adec647eeb09606a163a4c4c08fa5fff51ab3f08","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":3,"id":"89bdb25f_4aad013b","updated":"2021-11-08 14:30:19.000000000","message":"Doc formatting comments inline.  One of these days I will actually read the content.","commit_id":"1e8ee05d68d901779af1bef6f4160642a3909412"},{"author":{"_account_id":1736,"name":"Ivan Kolodyazhny","email":"e0ne@e0ne.info","username":"e0ne"},"change_message_id":"3f014023d538d79b96da2a4c93a749a8ef04ea04","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"6c25319a_728c1627","updated":"2021-11-14 18:08:53.000000000","message":"LGTM","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"490c4dc9056727bf4d00de1508d5004809613371","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"8e461c51_6aa1430a","updated":"2021-11-15 20:09:53.000000000","message":"LGTM","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":20813,"name":"Sofia Enriquez","email":"lsofia.enriquez@gmail.com","username":"enriquetaso"},"change_message_id":"f8c25b7a6b256f889c5bab3fc9c4964b55f68c82","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"1c1aea15_52cf4169","updated":"2021-11-15 21:21:12.000000000","message":"Looks good to me, waiting to see more comments for stable cores.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"cb2e7b6aedffe45c6c8f4b74cf1ff9cba70086f2","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"d6e6bfb8_e16196b6","updated":"2021-11-10 08:54:07.000000000","message":"Requesting reviewers attention please \u003c3","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"293a868d87332747700c4c88910ab2e043068a61","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"ebe3af3e_ef0946dc","updated":"2021-11-10 13:34:44.000000000","message":"Some nits noted inline for when you put up the next revision.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":4,"id":"bf2b214a_d87d7619","updated":"2021-12-02 12:05:18.000000000","message":"Thanks for working on this!","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":5,"id":"0880a7c5_23e663c3","updated":"2021-12-14 06:13:52.000000000","message":"Thank you for all the great reviews! Looks like we are ironing out the last bits for a clean spec!","commit_id":"2682c7b8637fbc127e39c3ac09e101bb08e290ad"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"d9fa3173e869899f22ab40a3b61861a9d0f9ef41","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":6,"id":"b8bc63df_1179c8b0","updated":"2021-12-14 14:03:33.000000000","message":"Fixed the RST, no change to content.","commit_id":"8257b17eca61ce9c656ea46ceb1528fb632a6b9e"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"c5e0dbd63bafeec33cee36c6dc126fe5fecf929b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"21fadf59_fbc2f959","updated":"2021-12-15 13:55:01.000000000","message":"Fixed the RST, but introduced a pep8 error.  Should be good now.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"76627c7cd45a828f8b0e62a383226acbc0ba45e6","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"9b8535a0_c0f2067d","updated":"2021-12-15 19:25:35.000000000","message":"I have one question inline.  I don\u0027t think it is a blocker though.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"bf53b6491b3ffd94c7eb7287ad7cebb3cd621c6b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"31f5dfc3_e73ab173","updated":"2021-12-20 17:15:57.000000000","message":"I think we\u0027re all in agreement that the spec is basically good.  Dropping my vote to +1 to give Zohar time to respond to Gorka\u0027s latest comments.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"578d341e4e8dce482099a53a3dd40f4ed67f2e7b","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"00d730bb_aa5a6538","updated":"2021-12-17 18:03:18.000000000","message":"LGTM.  I agree with Zohar\u0027s point that \"I don\u0027t think DM should be used as a fallback for nvmeof native multipath. I now think that the only fallback that should be allowed is dropping from multipathing to singlepathing rather than switching between mechanism.\"  I think this makes sense for the first implementation.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"8b956f73_52e1ecd7","updated":"2021-12-20 17:07:47.000000000","message":"The spec looks great in general, but I\u0027m not sure about:\n\n* Not supporting DM multipathing fallback (I\u0027m probably not fully understanding the NVMe implications)\n* How the code will behave with enforce multipathing","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":7,"id":"e7aa4539_fd823092","in_reply_to":"8b956f73_52e1ecd7","updated":"2021-12-21 11:43:52.000000000","message":"I think these two points are closely related, lets make the decision there.\n\nAnd thank you for the insights regarding volume identification details, specifically differences between existing os-brick implementation with iSCSI vs NVMe specs.\n\nnative-\u003eDM fallback - The way I see it now, based on volume id info, this would be possible in a situation where there is an (unintended) mismatch between all three: NGUID, EUI64, and Namespace ID. In that case, if all else is well (other hardware ID info is matching), DM may just take over automagically - maybe - and at which point the DM devices would need to be identified correctly. I think we haven\u0027t yet tested any of this os-brick NVMe multipath interaction with DM, so lets see if there aren\u0027t any complications even with the basic functionality.\n\nenforce_multipath - If we decide to support native-\u003eDM fallback, enfore_multipath\u003dTrue should first attempt this fallback, and fail the attachment if unsuccessful. enforce_multipath\u003dFalse should also attempt fallback to DM, and if unsuccessful, fallback to singlepath.\nIf we leave native-\u003eDM fallback out for now, enforce_multipath\u003dTrue will simply fail (instead of fallback to single path) when multipath connection fails.\n\nSo the decision is simply, support native-\u003eDM fallback or not?\n\nI vote to leave it out from this implementation, due to differences in volume identification details, I think unexpected things can get ugly. I think there is a lot of real hardware testing needed to make a confident decision on doing it. And if left out, it can always be added later. 😊\nIf we do go with native-\u003eDM fallback, I think it would be better to leave the opinions to the users/operators, so rather than assuming this native-\u003efallback, it should be configurable in another parameter.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"8912b8435a765f197616bcb672eb4b73a6de7edd","unresolved":true,"context_lines":[],"source_content_type":"","patch_set":7,"id":"46400d03_4f54ea05","in_reply_to":"e7aa4539_fd823092","updated":"2021-12-21 11:46:25.000000000","message":"Oops, did not mean to mark this resolved... Unresolving.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"da5a46870f2f74ddfe03d813b200f0b2824d49d5","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"11722a32_15faf89d","updated":"2021-12-21 15:49:05.000000000","message":"Everything looks very clear, except the enforcing section, that I think still needs a bit of clarification.","commit_id":"cc3a24cca4e9ff02f10d3917d2314e29081345d1"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":8,"id":"8dc4acc0_deb03057","updated":"2021-12-21 11:43:52.000000000","message":"Thank you for the review, the discussion continues!","commit_id":"cc3a24cca4e9ff02f10d3917d2314e29081345d1"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"8d6c7cba5aa6f8fad95f2428d7dfb4598752d944","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"2eb1071a_2e6ad0e7","updated":"2021-12-22 04:21:00.000000000","message":"I think my understanding of how enforce_multipath works is cleared up now.\n\nMade changes in 3 different locations where it was mentioned.\n\nPlease take a look at latest revision diff if the changes make sense.\n\nThank you!","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"ab5273465e5be58b64ed7851a93629185cc491da","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"1f989472_7ecfbf13","updated":"2021-12-22 16:44:06.000000000","message":"Revisions LGTM.","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"7aad36d1010e20896458fe346285d62b475d58d1","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"06446e39_ce8434ed","updated":"2021-12-23 10:14:28.000000000","message":"Zohar thanks for your patience with the spec.  Look great now.","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"006539334829f5c9416df7371183eab929b653dd","unresolved":false,"context_lines":[],"source_content_type":"","patch_set":9,"id":"5ccea839_90279a3f","updated":"2021-12-22 21:10:52.000000000","message":"looks good to me!","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"}],"specs/yoga/nvme-multipath.rst":[{"author":{"_account_id":27615,"name":"Rajat Dhasmana","email":"rajatdhasmana@gmail.com","username":"whoami-rajat"},"change_message_id":"9d6c7406ca93c096abeabc29c5fa4117d6722c5d","unresolved":true,"context_lines":[{"line_number":33,"context_line":""},{"line_number":34,"context_line":"NVMeoF ANA (Asynchronous Namespace Access) is the NVMeoF native way of doing"},{"line_number":35,"context_line":"multipathing, and it offers an interesting set of features."},{"line_number":36,"context_line":"A spec is available for more under-the-hood details (link under references)"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."}],"source_content_type":"text/x-rst","patch_set":1,"id":"a3fb1f01_40535abb","line":36,"range":{"start_line":36,"start_character":52,"end_line":36,"end_character":75},"updated":"2021-11-02 05:13:43.000000000","message":"would be better to add link to the reference like [1]_\nand in references, .. [1] \u003clink\u003e\nlike done here https://review.opendev.org/c/openstack/cinder-specs/+/810363/7/specs/yoga/optimize-upload-volume-to-rbd-store.rst#23\nL#23 and L#167","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":33,"context_line":""},{"line_number":34,"context_line":"NVMeoF ANA (Asynchronous Namespace Access) is the NVMeoF native way of doing"},{"line_number":35,"context_line":"multipathing, and it offers an interesting set of features."},{"line_number":36,"context_line":"A spec is available for more under-the-hood details (link under references)"},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."}],"source_content_type":"text/x-rst","patch_set":1,"id":"948faef8_ed449d82","line":36,"range":{"start_line":36,"start_character":52,"end_line":36,"end_character":75},"in_reply_to":"a3fb1f01_40535abb","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"},{"line_number":52,"context_line":"connection is established to it. If one of the paths breaks (such as in a"},{"line_number":53,"context_line":"partial network failure) - the attached device would remain functional,"},{"line_number":54,"context_line":"whereas with a single pathed connections, the device would become unavailable"},{"line_number":55,"context_line":"to the host it was attached to."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"96221598_fd8e35a9","line":54,"range":{"start_line":54,"start_character":0,"end_line":54,"end_character":40},"updated":"2021-11-02 16:46:21.000000000","message":"whereas with single pathed connections","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"},{"line_number":52,"context_line":"connection is established to it. If one of the paths breaks (such as in a"},{"line_number":53,"context_line":"partial network failure) - the attached device would remain functional,"},{"line_number":54,"context_line":"whereas with a single pathed connections, the device would become unavailable"},{"line_number":55,"context_line":"to the host it was attached to."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"54820f89_144731d6","line":54,"range":{"start_line":54,"start_character":0,"end_line":54,"end_character":40},"in_reply_to":"96221598_fd8e35a9","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"},{"line_number":73,"context_line":"multiple portals for the same NVMeoF device. Currently only the first"},{"line_number":74,"context_line":"successful connection is established."}],"source_content_type":"text/x-rst","patch_set":1,"id":"1988cfcc_075996f4","line":71,"updated":"2021-11-02 16:46:21.000000000","message":"NIT:  extra blank line here.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":""},{"line_number":72,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"},{"line_number":73,"context_line":"multiple portals for the same NVMeoF device. Currently only the first"},{"line_number":74,"context_line":"successful connection is established."}],"source_content_type":"text/x-rst","patch_set":1,"id":"a895e11b_e3b8d95f","line":71,"in_reply_to":"1988cfcc_075996f4","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":12670,"name":"Helen Walsh","email":"helen.walsh@emc.com","username":"walshh2"},"change_message_id":"257464edc9480407d04d17b990b815dc98e00766","unresolved":true,"context_lines":[{"line_number":74,"context_line":"successful connection is established."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"In multipath mode, the portals connection method should simply connect to"},{"line_number":77,"context_line":"all portals buy running the `nvme connect` cli command for each one."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"The NVMe system under the hood will create invisible devices for each of the"},{"line_number":80,"context_line":"portal connections, and expose a single multipath device on top of them."}],"source_content_type":"text/x-rst","patch_set":1,"id":"95fa2085_9287c527","line":77,"updated":"2021-11-02 11:26:27.000000000","message":"nit: by instead of buy","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":74,"context_line":"successful connection is established."},{"line_number":75,"context_line":""},{"line_number":76,"context_line":"In multipath mode, the portals connection method should simply connect to"},{"line_number":77,"context_line":"all portals buy running the `nvme connect` cli command for each one."},{"line_number":78,"context_line":""},{"line_number":79,"context_line":"The NVMe system under the hood will create invisible devices for each of the"},{"line_number":80,"context_line":"portal connections, and expose a single multipath device on top of them."}],"source_content_type":"text/x-rst","patch_set":1,"id":"e332287a_274ddbe0","line":77,"in_reply_to":"95fa2085_9287c527","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":84,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":85,"context_line":"OpenStack code)"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":89,"context_line":"in the baseline single path connector."},{"line_number":90,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"4eec0cb9_3e2bc405","line":87,"updated":"2021-11-02 16:46:21.000000000","message":"Nit:  Extra blank line.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":84,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":85,"context_line":"OpenStack code)"},{"line_number":86,"context_line":""},{"line_number":87,"context_line":""},{"line_number":88,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":89,"context_line":"in the baseline single path connector."},{"line_number":90,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"6c31ba7f_75ba68df","line":87,"in_reply_to":"4eec0cb9_3e2bc405","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"754fa5d91468d73ad85fe0aeb2d128a97999ee4e","unresolved":true,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"19e0647f_53044ad7","line":94,"updated":"2021-11-01 18:15:37.000000000","message":"To add future-proofing for synchronous cross-array replication, we should make this a list. Even in the first implementation, the list can be a single NQN, but it would be better to have this as a list now rather than change it later. This would bring it in line with the iSCSI and FC connector implementation","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"53065bef653643536039595681a893532bf66aa8","unresolved":false,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"6e4a7b2e_52f5da02","line":94,"in_reply_to":"0d8d6e21_8f3ba9e2","updated":"2021-11-03 12:19:35.000000000","message":"So does the spec state that a volume with a unique UUID can have multiple NQNs associated with it and when the `connect` is made it will perform the connect across all paths to the same UUID no matter what the underlying NQNs even though you are only providing one NQN as a connection reference point? To me, this doesn\u0027t feel correct, but I haven\u0027t read the spec","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"cf60d06c6bb9a5ba71300d8fb46c390a06d37061","unresolved":false,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"77f9c9df_0d3d0863","line":94,"in_reply_to":"19e0647f_53044ad7","updated":"2021-11-02 04:15:43.000000000","message":"Please refer to phase 2 (DM functionality, which is what should be in-line with iSCSI and FC)\nIt specifically mentions allowing for multiple NQNs on line 116, and you can see the example at line 130.\n\nWhat you are pointing at here is a single native multipathed connection, without sync replication or DM. According to NVMe spec, it can only ever have one NQN.\n\nResolving this.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"c16929d92ac62f399153474fb2686726aa3ffd35","unresolved":false,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"30dca908_acbe096e","line":94,"in_reply_to":"6e4a7b2e_52f5da02","updated":"2021-11-04 05:16:09.000000000","message":"Yes, it allows that by letting you freely specify NQN and UUID in any configuration for the volume copies/replicas (you can see in the connection information spec) such that you can specify the same UUID and different NQNs for your replicas and connection will be made on all of them (and then it will be up to DeviceMapper to match and aggregate them)","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"5becbc007bbf4131ee5a4b0ef5e5c10ca3fd2daa","unresolved":false,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"d611aeb6_ebce4bab","line":94,"in_reply_to":"77f9c9df_0d3d0863","updated":"2021-11-02 04:25:10.000000000","message":"Your DM multipath spec assumes that DM will multipath across two devices with different UUIDs.\nThis is not the case I am referring to.\nIn a fully synchronous, cross-array replication, you will have an active/active copy of a volume on two arrays at the same time with the same UUID. The volume is read/write capable on both arrays simultaneously, however, the two arrays will have different NQNs, therefore the connections for the native device will have to be to two different NQNs.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":91,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":92,"context_line":"{"},{"line_number":93,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":94,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":95,"context_line":"    \u0027portals\u0027: ["},{"line_number":96,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":97,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"0d8d6e21_8f3ba9e2","line":94,"in_reply_to":"d611aeb6_ebce4bab","updated":"2021-11-03 08:45:52.000000000","message":"That is completely fine and compatible with the spec. There is no assumption that UUIDs of replicas need to be different.\n\nThough I did just notice that the DM spec was missing the per-replica UUID part, adding it in.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":12670,"name":"Helen Walsh","email":"helen.walsh@emc.com","username":"walshh2"},"change_message_id":"257464edc9480407d04d17b990b815dc98e00766","unresolved":true,"context_lines":[{"line_number":123,"context_line":"Once connections are established, DeviceMapper will handle the multipathing"},{"line_number":124,"context_line":"under the hood (should be transparent to OpenStack code)"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Since DeviceMapper recognizes two devices should be \"merged\" based on device"},{"line_number":127,"context_line":"id information (udev?) - it is the responsiblity of the backend to handle its"},{"line_number":128,"context_line":"exposed devices to conform to the requirement."},{"line_number":129,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"ef753a7d_38a85acf","line":126,"updated":"2021-11-02 11:26:27.000000000","message":"Since DeviceMapper recognizes two devices, they should be \"merged\" ...","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":true,"context_lines":[{"line_number":123,"context_line":"Once connections are established, DeviceMapper will handle the multipathing"},{"line_number":124,"context_line":"under the hood (should be transparent to OpenStack code)"},{"line_number":125,"context_line":""},{"line_number":126,"context_line":"Since DeviceMapper recognizes two devices should be \"merged\" based on device"},{"line_number":127,"context_line":"id information (udev?) - it is the responsiblity of the backend to handle its"},{"line_number":128,"context_line":"exposed devices to conform to the requirement."},{"line_number":129,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"45099dc7_479a49db","line":126,"in_reply_to":"ef753a7d_38a85acf","updated":"2021-11-03 08:45:52.000000000","message":"I think I made a run-on sentence here, but the intent is to say that two devices are matched based on some information, then they should be merged. I try to reword this part here to make it more clear","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":154,"context_line":""},{"line_number":155,"context_line":"With this proposal, there will be multiple replication and/or multipath mode"},{"line_number":156,"context_line":"permutations"},{"line_number":157,"context_line":"of operation for the connector:"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"- Single pathed connection"},{"line_number":160,"context_line":"- Native multipathed connection"}],"source_content_type":"text/x-rst","patch_set":1,"id":"c6374d6e_92fd8cc0","line":157,"updated":"2021-11-02 16:46:21.000000000","message":"Nit:  Remove newline here.","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":154,"context_line":""},{"line_number":155,"context_line":"With this proposal, there will be multiple replication and/or multipath mode"},{"line_number":156,"context_line":"permutations"},{"line_number":157,"context_line":"of operation for the connector:"},{"line_number":158,"context_line":""},{"line_number":159,"context_line":"- Single pathed connection"},{"line_number":160,"context_line":"- Native multipathed connection"}],"source_content_type":"text/x-rst","patch_set":1,"id":"a430bae0_26020ffb","line":157,"in_reply_to":"c6374d6e_92fd8cc0","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"754fa5d91468d73ad85fe0aeb2d128a97999ee4e","unresolved":true,"context_lines":[{"line_number":164,"context_line":"Single pathed connection:"},{"line_number":165,"context_line":"{"},{"line_number":166,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":167,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":168,"context_line":"    \u0027portals\u0027: [(\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":169,"context_line":"}"},{"line_number":170,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"014ed421_c9bd7a0b","line":167,"updated":"2021-11-01 18:15:37.000000000","message":"Make list as above","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"cf60d06c6bb9a5ba71300d8fb46c390a06d37061","unresolved":false,"context_lines":[{"line_number":164,"context_line":"Single pathed connection:"},{"line_number":165,"context_line":"{"},{"line_number":166,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":167,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":168,"context_line":"    \u0027portals\u0027: [(\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":169,"context_line":"}"},{"line_number":170,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5b53a715_0a48e28d","line":167,"in_reply_to":"014ed421_c9bd7a0b","updated":"2021-11-02 04:15:43.000000000","message":"same as comment on line 94","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":179,"context_line":"}"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"For RAID and DeviceMapper, the example below is given with no native"},{"line_number":182,"context_line":"multipath for the volume replicas devices."},{"line_number":183,"context_line":"For these replica devices to do native multipathing, portals for each of"},{"line_number":184,"context_line":"each device\u0027s paths should be specified in the `portals` list."},{"line_number":185,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"5e4f0255_89def074","line":182,"range":{"start_line":182,"start_character":25,"end_line":182,"end_character":33},"updated":"2021-11-02 16:46:21.000000000","message":"replica","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":179,"context_line":"}"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"For RAID and DeviceMapper, the example below is given with no native"},{"line_number":182,"context_line":"multipath for the volume replicas devices."},{"line_number":183,"context_line":"For these replica devices to do native multipathing, portals for each of"},{"line_number":184,"context_line":"each device\u0027s paths should be specified in the `portals` list."},{"line_number":185,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"2c1fe3cb_1d53727b","line":182,"range":{"start_line":182,"start_character":25,"end_line":182,"end_character":33},"in_reply_to":"5e4f0255_89def074","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"100fdbb5aa2aae37d20d6db95a28a9551eed4fae","unresolved":true,"context_lines":[{"line_number":185,"context_line":""},{"line_number":186,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":187,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":188,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replicas"},{"line_number":189,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DevMapper and"},{"line_number":190,"context_line":"the replica devices are left hanging, how would we recognize this? And would"},{"line_number":191,"context_line":"we want to fail back to using RAID here instead?)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"504a6a75_463fdbcb","line":188,"range":{"start_line":188,"start_character":70,"end_line":188,"end_character":78},"updated":"2021-11-02 16:46:21.000000000","message":"replica","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":185,"context_line":""},{"line_number":186,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":187,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":188,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replicas"},{"line_number":189,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DevMapper and"},{"line_number":190,"context_line":"the replica devices are left hanging, how would we recognize this? And would"},{"line_number":191,"context_line":"we want to fail back to using RAID here instead?)"}],"source_content_type":"text/x-rst","patch_set":1,"id":"c2f1b39a_9b5cdc58","line":188,"range":{"start_line":188,"start_character":70,"end_line":188,"end_character":78},"in_reply_to":"504a6a75_463fdbcb","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":12670,"name":"Helen Walsh","email":"helen.walsh@emc.com","username":"walshh2"},"change_message_id":"257464edc9480407d04d17b990b815dc98e00766","unresolved":true,"context_lines":[{"line_number":186,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":187,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":188,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replicas"},{"line_number":189,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DevMapper and"},{"line_number":190,"context_line":"the replica devices are left hanging, how would we recognize this? And would"},{"line_number":191,"context_line":"we want to fail back to using RAID here instead?)"},{"line_number":192,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"6a443f98_902f05c2","line":189,"updated":"2021-11-02 11:26:27.000000000","message":"which does not trigger the DeviceMapper?","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"03e9faa8c54906424fc39ea0af24239fde73b2e1","unresolved":false,"context_lines":[{"line_number":186,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":187,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":188,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replicas"},{"line_number":189,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DevMapper and"},{"line_number":190,"context_line":"the replica devices are left hanging, how would we recognize this? And would"},{"line_number":191,"context_line":"we want to fail back to using RAID here instead?)"},{"line_number":192,"context_line":""}],"source_content_type":"text/x-rst","patch_set":1,"id":"76c8c8ac_28e507b3","line":189,"in_reply_to":"6a443f98_902f05c2","updated":"2021-11-03 08:45:52.000000000","message":"Done","commit_id":"dc71077ee248decd3cfc2f3d6ea1794223dc7367"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"b11ecae0a8d2b26507da4bd16b7702e167c19986","unresolved":true,"context_lines":[{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":90,"context_line":"{"},{"line_number":91,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":92,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":93,"context_line":"    \u0027portals\u0027: ["},{"line_number":94,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":95,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"},{"line_number":96,"context_line":"    ]"},{"line_number":97,"context_line":"}"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"In the case where the host does not support multipathing but the"},{"line_number":100,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"}],"source_content_type":"text/x-rst","patch_set":2,"id":"6a30bf13_2d163b3d","line":97,"range":{"start_line":89,"start_character":0,"end_line":97,"end_character":1},"updated":"2021-11-05 12:59:59.000000000","message":"This is making the docs job barf.  Add\n\n  .. code-block:: json\n\nbefore lines 90-97, and then indent those lines 4 spaces.\n\nMake a similar change for the other json examples below.","commit_id":"7459a1bd5fbf302d95a09c2ef0114191af072904"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"c23bbad3d31ece1c72b040b7da136d6872124cd3","unresolved":false,"context_lines":[{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":90,"context_line":"{"},{"line_number":91,"context_line":"    \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":92,"context_line":"    \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":93,"context_line":"    \u0027portals\u0027: ["},{"line_number":94,"context_line":"        (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":95,"context_line":"        (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"},{"line_number":96,"context_line":"    ]"},{"line_number":97,"context_line":"}"},{"line_number":98,"context_line":""},{"line_number":99,"context_line":"In the case where the host does not support multipathing but the"},{"line_number":100,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"}],"source_content_type":"text/x-rst","patch_set":2,"id":"85e07867_3ef4d19b","line":97,"range":{"start_line":89,"start_character":0,"end_line":97,"end_character":1},"in_reply_to":"6a30bf13_2d163b3d","updated":"2021-11-08 11:17:16.000000000","message":"Thank you for this!\n\nIt was not really clear for me from the zuul output what was the change that needed to be made, thank you for the help.","commit_id":"7459a1bd5fbf302d95a09c2ef0114191af072904"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"adec647eeb09606a163a4c4c08fa5fff51ab3f08","unresolved":true,"context_lines":[{"line_number":87,"context_line":"in the baseline single path connector."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":90,"context_line":"  .. code-block:: json"},{"line_number":91,"context_line":"    {"},{"line_number":92,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":93,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"63874de4_4433830b","line":90,"range":{"start_line":90,"start_character":0,"end_line":90,"end_character":22},"updated":"2021-11-08 14:30:19.000000000","message":"sorry, I gave you a bum steer here.  This isn\u0027t valid JSON, it\u0027s python (I saw API at line 71 and was thinking REST API, but that\u0027s not what you\u0027re working with here).\n\nAnyway, two things to fix these:\n\n1. change \u0027json\u0027 to \u0027python\u0027 in the code-block directive\n2. there has to be a blank line after the code-block directive before the code starts","commit_id":"1e8ee05d68d901779af1bef6f4160642a3909412"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"32707fe1ec38f55875263a2d44bcdf6d17b5257c","unresolved":false,"context_lines":[{"line_number":87,"context_line":"in the baseline single path connector."},{"line_number":88,"context_line":""},{"line_number":89,"context_line":"The connection information for a native multipath volume will look like:"},{"line_number":90,"context_line":"  .. code-block:: json"},{"line_number":91,"context_line":"    {"},{"line_number":92,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":93,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"}],"source_content_type":"text/x-rst","patch_set":3,"id":"7eafa45f_90787b7b","line":90,"range":{"start_line":90,"start_character":0,"end_line":90,"end_character":22},"in_reply_to":"63874de4_4433830b","updated":"2021-11-09 09:04:07.000000000","message":"Done","commit_id":"1e8ee05d68d901779af1bef6f4160642a3909412"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"adec647eeb09606a163a4c4c08fa5fff51ab3f08","unresolved":true,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"References"},{"line_number":353,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":354,"context_line":""},{"line_number":355,"context_line":"ANA Spec - Section 8.1.3"},{"line_number":356,"context_line":".. [1] https://nvmexpress.org/wp-content/uploads/NVMe-NVM-Express-2.0a-2021.07.26-Ratified.pdf"}],"source_content_type":"text/x-rst","patch_set":3,"id":"58966117_172d6c9e","line":356,"range":{"start_line":354,"start_character":0,"end_line":356,"end_character":94},"updated":"2021-11-08 14:30:19.000000000","message":"not exactly sure why it doesn\u0027t like this, possible the parser believes these lines are all part of the same paragraph, which would put the reference creation in a weird place, since it\u0027s supposed to be at the beginning of the line.  if you change it like this, it will work:\n\n  .. [1] ANA Spec - Section 8.1.3: https://nvmexpress.org/wp-content/uploads/NVMe-NVM-Express-2.0a-2021.07.26-Ratified.pdf","commit_id":"1e8ee05d68d901779af1bef6f4160642a3909412"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"32707fe1ec38f55875263a2d44bcdf6d17b5257c","unresolved":false,"context_lines":[{"line_number":351,"context_line":""},{"line_number":352,"context_line":"References"},{"line_number":353,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":354,"context_line":""},{"line_number":355,"context_line":"ANA Spec - Section 8.1.3"},{"line_number":356,"context_line":".. [1] https://nvmexpress.org/wp-content/uploads/NVMe-NVM-Express-2.0a-2021.07.26-Ratified.pdf"}],"source_content_type":"text/x-rst","patch_set":3,"id":"a2f85f78_4009e3a7","line":356,"range":{"start_line":354,"start_character":0,"end_line":356,"end_character":94},"in_reply_to":"58966117_172d6c9e","updated":"2021-11-09 09:04:07.000000000","message":"Done","commit_id":"1e8ee05d68d901779af1bef6f4160642a3909412"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"NVMeoF Multipathing"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cinder/+spec/nvmeof-multipath"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6e750357_daec2444","line":8,"updated":"2021-12-02 12:05:18.000000000","message":"I think we should mention that at the time of writing this spec only TCP and RDMA protocols are supported by os-brick, so the spec doesn\u0027t contemplate the case for other transport protocols such as FC.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":5,"context_line":" http://creativecommons.org/licenses/by/3.0/legalcode"},{"line_number":6,"context_line":""},{"line_number":7,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":8,"context_line":"NVMeoF Multipathing"},{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cinder/+spec/nvmeof-multipath"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6ca5dc08_7e568aeb","line":8,"in_reply_to":"6e750357_daec2444","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cinder/+spec/nvmeof-multipath"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Extend the NVMeoF connector to support multipathing."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This will be broken up into two phases:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Phase 1 - Native NVMeoF multipathing"},{"line_number":18,"context_line":"Phase 2 - Use of DeviceMapper for multipathing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1f53aa20_d40be3ca","line":15,"range":{"start_line":12,"start_character":0,"end_line":15,"end_character":39},"updated":"2021-12-02 12:05:18.000000000","message":"In this section we should just talk about what the spec is about, not go over the the division of work.\n\nSo it would be something like what you have but phrased differently: \"Extend the NVMeoF connector to support multipathing it all its versions, using a device mapper multipath and with the native NVMeoF multipathing (ANA).","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":9,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":10,"context_line":""},{"line_number":11,"context_line":"https://blueprints.launchpad.net/cinder/+spec/nvmeof-multipath"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"Extend the NVMeoF connector to support multipathing."},{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This will be broken up into two phases:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Phase 1 - Native NVMeoF multipathing"},{"line_number":18,"context_line":"Phase 2 - Use of DeviceMapper for multipathing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6d973d6f_0d8aa9ab","line":15,"range":{"start_line":12,"start_character":0,"end_line":15,"end_character":39},"in_reply_to":"1f53aa20_d40be3ca","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"293a868d87332747700c4c88910ab2e043068a61","unresolved":true,"context_lines":[{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This will be broken up into two phases:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Phase 1 - Native NVMeoF multipathing"},{"line_number":18,"context_line":"Phase 2 - Use of DeviceMapper for multipathing"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5c0cb6b2_f8070647","line":18,"range":{"start_line":17,"start_character":0,"end_line":18,"end_character":46},"updated":"2021-11-10 13:34:44.000000000","message":"sphinx is making these into a single paragraph.  probably best to turn them into a bulleted list (put a \u0027* \u0027 at the beginning of each line).","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":14,"context_line":""},{"line_number":15,"context_line":"This will be broken up into two phases:"},{"line_number":16,"context_line":""},{"line_number":17,"context_line":"Phase 1 - Native NVMeoF multipathing"},{"line_number":18,"context_line":"Phase 2 - Use of DeviceMapper for multipathing"},{"line_number":19,"context_line":""},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2881d524_537ef548","line":18,"range":{"start_line":17,"start_character":0,"end_line":18,"end_character":46},"in_reply_to":"5c0cb6b2_f8070647","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":false,"context_lines":[{"line_number":24,"context_line":"Currently the NVMeoF connector only supports single pathed connections."},{"line_number":25,"context_line":""},{"line_number":26,"context_line":"The problem description can be summed up by the statement:"},{"line_number":27,"context_line":"A single pathed connection is a single point of failure."},{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For resiliency, and to catch up with industry standards, support for"},{"line_number":30,"context_line":"multi pathed connections is needed, mitigating the single point of failure."}],"source_content_type":"text/x-rst","patch_set":4,"id":"9b3fcf75_afca13ff","line":27,"range":{"start_line":27,"start_character":0,"end_line":27,"end_character":56},"updated":"2021-12-02 12:05:18.000000000","message":"Nicely summarized. 😊","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For resiliency, and to catch up with industry standards, support for"},{"line_number":30,"context_line":"multi pathed connections is needed, mitigating the single point of failure."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"This can be done in two ways, which could even be used together:"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"NVMeoF ANA (Asynchronous Namespace Access) is the NVMeoF native way of doing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3aad6f14_bd8fd356","line":31,"range":{"start_line":31,"start_character":0,"end_line":31,"end_character":0},"updated":"2021-12-02 12:05:18.000000000","message":"nit: We should also mention potential performance benefits and replication, because if I remember correctly some drivers (e.g. Pure) does synchronous replication and they return the locations for the mirrors so that multipath DM can switch on failure.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":28,"context_line":""},{"line_number":29,"context_line":"For resiliency, and to catch up with industry standards, support for"},{"line_number":30,"context_line":"multi pathed connections is needed, mitigating the single point of failure."},{"line_number":31,"context_line":""},{"line_number":32,"context_line":"This can be done in two ways, which could even be used together:"},{"line_number":33,"context_line":""},{"line_number":34,"context_line":"NVMeoF ANA (Asynchronous Namespace Access) is the NVMeoF native way of doing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9a506499_6005bd23","line":31,"range":{"start_line":31,"start_character":0,"end_line":31,"end_character":0},"in_reply_to":"3aad6f14_bd8fd356","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":35,"context_line":"multipathing, and it offers an interesting set of features."},{"line_number":36,"context_line":"A spec is available for more under-the-hood details [1]_."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."},{"line_number":40,"context_line":"The goal of Phase 2 of this spec\u0027s implementation is to bring the NVMeoF"},{"line_number":41,"context_line":"connector up to speed with using DeviceMapper for multi pathed connections."}],"source_content_type":"text/x-rst","patch_set":4,"id":"e7082cc3_796d9d55","line":38,"updated":"2021-12-02 12:05:18.000000000","message":"We should mention that NVMeoF ANA support was not part of the initial standard, so there may be systems (both clients and storage systems) that don\u0027t support it and where the only possibility of doing multipathing was using the device mapper.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":35,"context_line":"multipathing, and it offers an interesting set of features."},{"line_number":36,"context_line":"A spec is available for more under-the-hood details [1]_."},{"line_number":37,"context_line":""},{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."},{"line_number":40,"context_line":"The goal of Phase 2 of this spec\u0027s implementation is to bring the NVMeoF"},{"line_number":41,"context_line":"connector up to speed with using DeviceMapper for multi pathed connections."}],"source_content_type":"text/x-rst","patch_set":4,"id":"95120918_2650cfe1","line":38,"in_reply_to":"e7082cc3_796d9d55","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."},{"line_number":40,"context_line":"The goal of Phase 2 of this spec\u0027s implementation is to bring the NVMeoF"},{"line_number":41,"context_line":"connector up to speed with using DeviceMapper for multi pathed connections."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2653d407_21fefe1b","line":41,"range":{"start_line":41,"start_character":50,"end_line":41,"end_character":62},"updated":"2021-12-02 12:05:18.000000000","message":"multipathed","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":38,"context_line":"Using DeviceMapper for multipathing is an industry standard approach with"},{"line_number":39,"context_line":"connections such as iSCSI, and can be equally utilized with NVMeoF connections."},{"line_number":40,"context_line":"The goal of Phase 2 of this spec\u0027s implementation is to bring the NVMeoF"},{"line_number":41,"context_line":"connector up to speed with using DeviceMapper for multi pathed connections."},{"line_number":42,"context_line":""},{"line_number":43,"context_line":""},{"line_number":44,"context_line":"Use Cases"}],"source_content_type":"text/x-rst","patch_set":4,"id":"bde64810_028fc816","line":41,"range":{"start_line":41,"start_character":50,"end_line":41,"end_character":62},"in_reply_to":"2653d407_21fefe1b","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":45,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Whether via native multipath or DeviceMapper, the use case is the ability to"},{"line_number":48,"context_line":"expose multi pathed connections to volumes, mitigating connection single point"},{"line_number":49,"context_line":"of failure."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"}],"source_content_type":"text/x-rst","patch_set":4,"id":"17e7d956_a8e40b66","line":48,"range":{"start_line":48,"start_character":7,"end_line":48,"end_character":19},"updated":"2021-12-02 12:05:18.000000000","message":"multipathed","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":45,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Whether via native multipath or DeviceMapper, the use case is the ability to"},{"line_number":48,"context_line":"expose multi pathed connections to volumes, mitigating connection single point"},{"line_number":49,"context_line":"of failure."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3e741e3c_b95ec6b3","line":48,"range":{"start_line":48,"start_character":7,"end_line":48,"end_character":19},"in_reply_to":"17e7d956_a8e40b66","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Whether via native multipath or DeviceMapper, the use case is the ability to"},{"line_number":48,"context_line":"expose multi pathed connections to volumes, mitigating connection single point"},{"line_number":49,"context_line":"of failure."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"},{"line_number":52,"context_line":"connection is established to it. If one of the paths breaks (such as in a"}],"source_content_type":"text/x-rst","patch_set":4,"id":"fb3c57b3_e2288753","line":49,"updated":"2021-12-02 12:05:18.000000000","message":"and improving performance","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":46,"context_line":""},{"line_number":47,"context_line":"Whether via native multipath or DeviceMapper, the use case is the ability to"},{"line_number":48,"context_line":"expose multi pathed connections to volumes, mitigating connection single point"},{"line_number":49,"context_line":"of failure."},{"line_number":50,"context_line":""},{"line_number":51,"context_line":"For example, an NVMeoF volume has two paths (portals) and a multipath"},{"line_number":52,"context_line":"connection is established to it. If one of the paths breaks (such as in a"}],"source_content_type":"text/x-rst","patch_set":4,"id":"d5fe0a25_3bae3269","line":49,"in_reply_to":"fb3c57b3_e2288753","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Proposed change"},{"line_number":59,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Phase 1 (Native):"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"The NVMeoF connector should check the host\u0027s multipath capability and report"}],"source_content_type":"text/x-rst","patch_set":4,"id":"cf1efcaa_775abdfd","line":60,"updated":"2021-12-02 12:05:18.000000000","message":"Please add a paragraph explaining the 2 phases (instead of explaining it at the beginning and why we are doing them in that order (I believe it was decided because the DM was meant as a fallback mechanism).","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":true,"context_lines":[{"line_number":57,"context_line":""},{"line_number":58,"context_line":"Proposed change"},{"line_number":59,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Phase 1 (Native):"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"The NVMeoF connector should check the host\u0027s multipath capability and report"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2be6eb4b_0d33d4bd","line":60,"in_reply_to":"cf1efcaa_775abdfd","updated":"2021-12-14 06:13:52.000000000","message":"DM in my opinion should not be meant as a fallback mechanism.\nThe multipath mechanism should be configurable by operator\u0027s choice (and system capabilities)\n\nThe order was chosen because KumoScale development is what is moving this effort forward, and in KumoScale\u0027s case native nvmeof multipath is a low hanging fruit that can actually be tested on their system. (While for nvmeof DM, whoever will want to support it such as Pure et al could also help with the implementation - though of course the spec contemplates both)","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Phase 1 (Native):"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"The NVMeoF connector should check the host\u0027s multipath capability and report"},{"line_number":64,"context_line":"it in the connector properties passed to the volume driver, this will also"},{"line_number":65,"context_line":"allow any driver / backend specific handling."},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"7823667f_8f7faf46","line":63,"range":{"start_line":63,"start_character":21,"end_line":63,"end_character":27},"updated":"2021-12-02 12:05:18.000000000","message":"will","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":60,"context_line":""},{"line_number":61,"context_line":"Phase 1 (Native):"},{"line_number":62,"context_line":""},{"line_number":63,"context_line":"The NVMeoF connector should check the host\u0027s multipath capability and report"},{"line_number":64,"context_line":"it in the connector properties passed to the volume driver, this will also"},{"line_number":65,"context_line":"allow any driver / backend specific handling."},{"line_number":66,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"bc02eb40_812072c8","line":63,"range":{"start_line":63,"start_character":21,"end_line":63,"end_character":27},"in_reply_to":"7823667f_8f7faf46","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":65,"context_line":"allow any driver / backend specific handling."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The value of `/sys/module/nvme_core/parameters/multipath` can be `Y` or `N` -"},{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"43e1aefe_df8efaa6","line":68,"range":{"start_line":68,"start_character":44,"end_line":68,"end_character":60},"updated":"2021-12-02 12:05:18.000000000","message":"-1: ``nvme_native_multipath`` or ``nvme_ana``","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":65,"context_line":"allow any driver / backend specific handling."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The value of `/sys/module/nvme_core/parameters/multipath` can be `Y` or `N` -"},{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"}],"source_content_type":"text/x-rst","patch_set":4,"id":"76b0440e_dc5c6fae","line":68,"range":{"start_line":68,"start_character":44,"end_line":68,"end_character":60},"in_reply_to":"43e1aefe_df8efaa6","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The value of `/sys/module/nvme_core/parameters/multipath` can be `Y` or `N` -"},{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"},{"line_number":72,"context_line":"multiple portals for the same NVMeoF device. Currently only the first"}],"source_content_type":"text/x-rst","patch_set":4,"id":"d98e5cf6_4b45306a","line":69,"updated":"2021-12-02 12:05:18.000000000","message":"the ``get_connector_properties`` method from the ``NVMeOFConnector`` connector class.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":66,"context_line":""},{"line_number":67,"context_line":"The value of `/sys/module/nvme_core/parameters/multipath` can be `Y` or `N` -"},{"line_number":68,"context_line":"this should be returned as a True or False `native_multipath` property in"},{"line_number":69,"context_line":"`get_connector_properties`"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The \"new\" NVMeoF connector API (introduced in Wallaby) supports passing"},{"line_number":72,"context_line":"multiple portals for the same NVMeoF device. Currently only the first"}],"source_content_type":"text/x-rst","patch_set":4,"id":"6c6ee953_9da75dec","line":69,"in_reply_to":"d98e5cf6_4b45306a","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"54132d5e_bb76c497","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"updated":"2021-12-02 12:05:18.000000000","message":"Maybe we should note that if they are different then os-brick will have to use the device mapper to do multipathing.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"82d358e2_39b1fa72","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"10377051_cbbbdf1d","updated":"2021-12-20 17:07:47.000000000","message":"The important thing with multipathing using iSCSI and FCP protocols is that both are running SCSI commands underneath, and the SCSI protocol defines the \"Device Identification VPD page (83h)\" which must remain the same through the life of a volume.  And I\u0027m sure that\u0027s what Pure storage is using to present the 2 different physical disks as one when doing replication.\n\nOur code in os-brick sometimes queries this information with the \"/lib/udev/scsi_id --page\u003d0x83 --whitelisted --device /dev/XYZ\" and other times uses the value in sysfs that the udev connect rules stored.\n\nI assumed that the NVMe protocol had something similar to uniquely identify a volume regardless of where or how it is being presented.\n\nLooking at various places [1][2][3] I see that this is supported by linux and exposed in sysfs under /dev/nvmeX/wwid, but that it depends on the presence of different data (was probably added in different NVMe spec versions). \n\nFrom highest to lowest preference, it seems to be [2]:\n- The NGUID\n- The EUI64\n- A concatenation of VID, Serial Number, Model and the Namespace ID\n\nSo, shouldn\u0027t we be able to support falling back to the DM option if the host does not support native multipathing?\n\n[1]: https://www.nvmexpress.org/wp-content/uploads/NVM-Express-SCSI-Translation-Reference-1_1-Gold.pdf\n[2]: http://lists.infradead.org/pipermail/linux-nvme/2016-February/004059.html\n[3]: https://lore.kernel.org/lkml/20170712133856.17211-1-jthumshirn@suse.de/","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"4537fe336cb66c8b84b200bd8a8fc5a0a3cc9b6d","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"4d7e3e98_5b932c4b","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"2e697f09_861a734b","updated":"2021-12-16 10:51:13.000000000","message":"That is a great point Simon, if there is only one physical device, there won\u0027t be a need to sync the hardware id info, since it will all be coming from the same device. (So I think no changes needed here for the spec unless Gorka has more feedback on this)\n\nA further question just for clarification for my own understanding:\n\nIn the case of synchronous replication, are there 2 physical devices? (in that case, syncing their hardware id info would be the responsibility of the backend - since openstack won\u0027t be doing it automagically)\n\nThank you!","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"5a604f34957d123a3611d2c45659d083b43a645a","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"10377051_cbbbdf1d","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"4d7e3e98_5b932c4b","updated":"2021-12-16 14:45:57.000000000","message":"For synchronous replication there would be two physicsal devixes. They would be provided by different backends, each with their own unique IP addresses got NVMe ports, or different FC ports if NVMe-FC.\nYes, the hardware wpould be responsible for syncing the hardware ID.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7cefab52_8e7df391","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"54132d5e_bb76c497","updated":"2021-12-14 06:13:52.000000000","message":"Two points about this:\n\n1. I don\u0027t think DM should be used as a fallback for nvmeof native multipath. I now think that the only fallback that should be allowed is dropping from multipathing to singlepathing rather than switching between mechanism.\n\n2. For DM - the storage backend is also responsible for assigning hardware id info for DM to recognize the seemingly two \"physical\" devices really being one - and therefore coupling them into one linux device. This is practically the same requirement as for native multipath except there it is NQN and namespace id - so, if nqn and namespace id are already messed up by the backend for (intended) multipathing to work, I doubt we should expect the DM-related backend requirements to be upheld (especially when unintended).\n\nEssetially, the way I see it now, operators should be clear on the multipath mechanism they intend on using. Moreover, there needs to be a clear compatibility between the backend\u0027s capabilities and the intended mechanism. (As is what I mean in both cases where mentioning \"the backend\u0027s responsibility\")\n\nAnd also, I now think that the only fallback that should be allowed is dropping from multipathing to singlepathing rather than switching between mechanism.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":13425,"name":"Simon Dodsley","email":"simon@purestorage.com","username":"sdodsley"},"change_message_id":"5b95e0d20eef63e4591a1811b9d4a482b1ef5010","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"2e697f09_861a734b","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"7cefab52_8e7df391","updated":"2021-12-14 13:09:13.000000000","message":"Not a -1, just a point of clarification:\n\nFor your response to point 2: In most external devices there will only be 1 device (I know Kumoscale uses 2). This device will, however, be presented to the host over multiple IP connections and therefore create the multiple connections for the single limux device.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":true,"context_lines":[{"line_number":80,"context_line":""},{"line_number":81,"context_line":"It should be noted that the native multipath spec requires the target portals"},{"line_number":82,"context_line":"to expose the device under the same target NQN and namespace id for multipath"},{"line_number":83,"context_line":"to happen. (This is the backend\u0027s responsibility and should be transparent to"},{"line_number":84,"context_line":"OpenStack code)"},{"line_number":85,"context_line":""},{"line_number":86,"context_line":"Finally, the new device will be matched by its uuid as is already implemented"},{"line_number":87,"context_line":"in the baseline single path connector."}],"source_content_type":"text/x-rst","patch_set":4,"id":"7dc847b4_3f28b0f8","line":84,"range":{"start_line":83,"start_character":11,"end_line":84,"end_character":15},"in_reply_to":"82d358e2_39b1fa72","updated":"2021-12-21 11:43:52.000000000","message":"Reply in the main (not-inline) comment thread, as I wanted to touch on both the fallback and enforce multipathing in one comment.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":20813,"name":"Sofia Enriquez","email":"lsofia.enriquez@gmail.com","username":"enriquetaso"},"change_message_id":"f8c25b7a6b256f889c5bab3fc9c4964b55f68c82","unresolved":true,"context_lines":[{"line_number":98,"context_line":"        ]"},{"line_number":99,"context_line":"    }"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"In the case where the host does not support multipathing but the"},{"line_number":102,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":103,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":104,"context_line":"is exposed."}],"source_content_type":"text/x-rst","patch_set":4,"id":"d4f68fea_7246554d","line":101,"range":{"start_line":101,"start_character":57,"end_line":101,"end_character":60},"updated":"2021-11-15 21:21:12.000000000","message":"Sorry if I\u0027m missing something but shouldn\u0027t this be \"and\" instead of but?","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":98,"context_line":"        ]"},{"line_number":99,"context_line":"    }"},{"line_number":100,"context_line":""},{"line_number":101,"context_line":"In the case where the host does not support multipathing but the"},{"line_number":102,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":103,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":104,"context_line":"is exposed."}],"source_content_type":"text/x-rst","patch_set":4,"id":"baf2dbed_2bcca1de","line":101,"range":{"start_line":101,"start_character":57,"end_line":101,"end_character":60},"in_reply_to":"d4f68fea_7246554d","updated":"2021-12-14 06:13:52.000000000","message":"Reading this again, \"and\" sounds much better than \"but\" - thank you.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":123,"context_line":"under the hood (should be transparent to OpenStack code)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"DeviceMapper needs to recognize that two devices should be \"merged\""},{"line_number":126,"context_line":"It does so based on device id information (udev?)"},{"line_number":127,"context_line":"Therefore, it is the responsibility of the backend to expose devices with"},{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."}],"source_content_type":"text/x-rst","patch_set":4,"id":"e87ee1e9_3d86c744","line":126,"range":{"start_line":126,"start_character":42,"end_line":126,"end_character":49},"updated":"2021-12-02 12:05:18.000000000","message":"I wouldn\u0027t mention udev, because the whole thing about how multipath notices new devices is non trivial, since there are udev rules but multipathd also has netlink uevents triggers from the kernel...","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":123,"context_line":"under the hood (should be transparent to OpenStack code)"},{"line_number":124,"context_line":""},{"line_number":125,"context_line":"DeviceMapper needs to recognize that two devices should be \"merged\""},{"line_number":126,"context_line":"It does so based on device id information (udev?)"},{"line_number":127,"context_line":"Therefore, it is the responsibility of the backend to expose devices with"},{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."}],"source_content_type":"text/x-rst","patch_set":4,"id":"6d0a062f_284a62bf","line":126,"range":{"start_line":126,"start_character":42,"end_line":126,"end_character":49},"in_reply_to":"e87ee1e9_3d86c744","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":125,"context_line":"DeviceMapper needs to recognize that two devices should be \"merged\""},{"line_number":126,"context_line":"It does so based on device id information (udev?)"},{"line_number":127,"context_line":"Therefore, it is the responsibility of the backend to expose devices with"},{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"When Since DeviceMapper recognizes two devices should be \"merged\" based on device"}],"source_content_type":"text/x-rst","patch_set":4,"id":"71204617_74abc4f4","line":128,"range":{"start_line":128,"start_character":3,"end_line":128,"end_character":26},"updated":"2021-12-02 12:05:18.000000000","message":"information is written twice","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":125,"context_line":"DeviceMapper needs to recognize that two devices should be \"merged\""},{"line_number":126,"context_line":"It does so based on device id information (udev?)"},{"line_number":127,"context_line":"Therefore, it is the responsibility of the backend to expose devices with"},{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"When Since DeviceMapper recognizes two devices should be \"merged\" based on device"}],"source_content_type":"text/x-rst","patch_set":4,"id":"a563c0cf_1318126f","line":128,"range":{"start_line":128,"start_character":3,"end_line":128,"end_character":26},"in_reply_to":"71204617_74abc4f4","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"When Since DeviceMapper recognizes two devices should be \"merged\" based on device"},{"line_number":132,"context_line":"id information (udev?) - it is the responsiblity of the backend to handle its"},{"line_number":133,"context_line":"exposed devices to conform to the requirement."},{"line_number":134,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"49ecee17_888c1056","line":131,"range":{"start_line":131,"start_character":0,"end_line":131,"end_character":10},"updated":"2021-12-02 12:05:18.000000000","message":"Since","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":128,"context_line":"id information information that conforms to the device matching requirements"},{"line_number":129,"context_line":"of DeviceMapper."},{"line_number":130,"context_line":""},{"line_number":131,"context_line":"When Since DeviceMapper recognizes two devices should be \"merged\" based on device"},{"line_number":132,"context_line":"id information (udev?) - it is the responsiblity of the backend to handle its"},{"line_number":133,"context_line":"exposed devices to conform to the requirement."},{"line_number":134,"context_line":""}],"source_content_type":"text/x-rst","patch_set":4,"id":"ee7f631a_f0c56bc9","line":131,"range":{"start_line":131,"start_character":0,"end_line":131,"end_character":10},"in_reply_to":"49ecee17_888c1056","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    {"},{"line_number":139,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":140,"context_line":"        \u0027volume_replicas\u0027: ["},{"line_number":141,"context_line":"            {"},{"line_number":142,"context_line":"                \u0027vol_uuid\u0027: \u0027...someuuid1...\u0027,"},{"line_number":143,"context_line":"                \u0027target_nqn\u0027: \u0027...somenqn1...\u0027,"}],"source_content_type":"text/x-rst","patch_set":4,"id":"e041f032_2599878e","line":140,"range":{"start_line":140,"start_character":9,"end_line":140,"end_character":24},"updated":"2021-12-02 12:05:18.000000000","message":"Reusing the \"volume_replicates\" key may be misleading, since these not need to be different replicas, they could just be different paths that point to the same disk.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":137,"context_line":""},{"line_number":138,"context_line":"    {"},{"line_number":139,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":140,"context_line":"        \u0027volume_replicas\u0027: ["},{"line_number":141,"context_line":"            {"},{"line_number":142,"context_line":"                \u0027vol_uuid\u0027: \u0027...someuuid1...\u0027,"},{"line_number":143,"context_line":"                \u0027target_nqn\u0027: \u0027...somenqn1...\u0027,"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1dd76a55_9670da5c","line":140,"range":{"start_line":140,"start_character":9,"end_line":140,"end_character":24},"in_reply_to":"e041f032_2599878e","updated":"2021-12-14 06:13:52.000000000","message":"Good idea, I will change it to \"dm_replicas\" for DM - this will also take care of your -1 on L158 - thank you!","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":151,"context_line":"        ]"},{"line_number":152,"context_line":"    }"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"Finally, the RAID replication mode feature (also part of the \"new\" connector"},{"line_number":155,"context_line":"API introduced in Wallaby) - also utilizes the `volume_replicas` field."},{"line_number":156,"context_line":"To avoid conflicts between the DeviceMapper and RAID modes of operation, a"},{"line_number":157,"context_line":"way to determine which mode to use (such as a flag) will be important to avoid"},{"line_number":158,"context_line":"engaging the RAID code paths when working in DeviceMapper replication mode."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Connector API Summary"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ac572c2e_68e0c243","line":158,"range":{"start_line":154,"start_character":0,"end_line":158,"end_character":75},"updated":"2021-12-02 12:05:18.000000000","message":"-1: Please be explicit about how Cinder drivers should report the raid behavior, as this detailed spec will serve as the documentation.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":151,"context_line":"        ]"},{"line_number":152,"context_line":"    }"},{"line_number":153,"context_line":""},{"line_number":154,"context_line":"Finally, the RAID replication mode feature (also part of the \"new\" connector"},{"line_number":155,"context_line":"API introduced in Wallaby) - also utilizes the `volume_replicas` field."},{"line_number":156,"context_line":"To avoid conflicts between the DeviceMapper and RAID modes of operation, a"},{"line_number":157,"context_line":"way to determine which mode to use (such as a flag) will be important to avoid"},{"line_number":158,"context_line":"engaging the RAID code paths when working in DeviceMapper replication mode."},{"line_number":159,"context_line":""},{"line_number":160,"context_line":""},{"line_number":161,"context_line":"Connector API Summary"}],"source_content_type":"text/x-rst","patch_set":4,"id":"03843a7d_2c0aa124","line":158,"range":{"start_line":154,"start_character":0,"end_line":158,"end_character":75},"in_reply_to":"ac572c2e_68e0c243","updated":"2021-12-14 06:13:52.000000000","message":"resolved by renaming DM replica list to \"dm_replicas\"","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"With this proposal, there will be multiple replication and/or multipath mode"},{"line_number":165,"context_line":"permutations"},{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"62847d82_98686d56","line":166,"range":{"start_line":165,"start_character":0,"end_line":166,"end_character":31},"updated":"2021-12-02 12:05:18.000000000","message":"nit: no need to be in separated lines (even if it will render the same)","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":162,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":163,"context_line":""},{"line_number":164,"context_line":"With this proposal, there will be multiple replication and/or multipath mode"},{"line_number":165,"context_line":"permutations"},{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"a2bab70f_f6cc9e6c","line":166,"range":{"start_line":165,"start_character":0,"end_line":166,"end_character":31},"in_reply_to":"62847d82_98686d56","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"a2c5800e_b84be6cf","line":169,"updated":"2021-12-02 12:05:18.000000000","message":"I assume RAID multipathed will only work with ANA, right? Since L154-L158 state that a flag will be used to know if the ``volume_replicas`` must be used for multipathing or RAID, so we wouldn\u0027t be able to have RAID with DM multipathing.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"5f5bd707_317536fe","line":169,"range":{"start_line":169,"start_character":29,"end_line":169,"end_character":41},"updated":"2021-12-02 12:05:18.000000000","message":"multipahted","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"d7cff0ef_18db5d9e","line":169,"in_reply_to":"22f1fc4a_f083b3e0","updated":"2021-12-21 11:43:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"dd3889fd_cb1061e3","line":169,"range":{"start_line":169,"start_character":29,"end_line":169,"end_character":41},"in_reply_to":"5f5bd707_317536fe","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"22f1fc4a_f083b3e0","line":169,"in_reply_to":"a0e0e004_6bd1c482","updated":"2021-12-20 17:07:47.000000000","message":"Seems reasonable.  Maybe say so in the spec?","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":true,"context_lines":[{"line_number":166,"context_line":"of operation for the connector:"},{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"}],"source_content_type":"text/x-rst","patch_set":4,"id":"a0e0e004_6bd1c482","line":169,"in_reply_to":"a2c5800e_b84be6cf","updated":"2021-12-14 06:13:52.000000000","message":"I think RAID on top of DM is overkill at the moment?\nIn order to allow this we will need to build a one level deeper nested tree for the devices (also overkill).","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"},{"line_number":173,"context_line":"  .. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":4,"id":"62df0460_ffb91e6f","line":170,"updated":"2021-12-02 12:05:18.000000000","message":"I think we are missing a combination where we have a multipath DM where each device is a multipathed ANA device.\n\nThis would probably be the case in systems like Pure when replication is enabled.  Each location will have multiple paths that can be connected using NVMe native multipathing, and each location will have different (target NQN, namespace id) so they will form different devices but with the same WWID so a DM multipath will be formed.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":false,"context_lines":[{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"},{"line_number":173,"context_line":"  .. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":4,"id":"ddda5ec3_48d3c062","line":170,"in_reply_to":"3056e227_dba47227","updated":"2021-12-20 17:07:47.000000000","message":"Ack","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":true,"context_lines":[{"line_number":167,"context_line":"- Single pathed connection"},{"line_number":168,"context_line":"- Native multipathed connection"},{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"},{"line_number":173,"context_line":"  .. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":4,"id":"3056e227_dba47227","line":170,"in_reply_to":"62df0460_ffb91e6f","updated":"2021-12-14 06:13:52.000000000","message":"This is exemplified by saying \"DM (single _or_ native multipath)\" - which expands out to:\n\nDM * (SP + MP) \u003d\nDM * SP +\nDM * MP\n:)\n\nI did this to save a bit of space for people not to get overwhelmed by all these structs, but I could expand it out more explicitly if you think that would be better.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"},{"line_number":173,"context_line":"  .. code-block:: python"},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"    {"},{"line_number":176,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":177,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":178,"context_line":"        \u0027portals\u0027: [(\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":179,"context_line":"    }"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"Native multipathed connection:"},{"line_number":182,"context_line":"  .. code-block:: python"},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"    {"},{"line_number":185,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":186,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":187,"context_line":"        \u0027portals\u0027: ["},{"line_number":188,"context_line":"            (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":189,"context_line":"            (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"},{"line_number":190,"context_line":"        ]"},{"line_number":191,"context_line":"    }"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"For RAID and DeviceMapper, the example below is given with no native"},{"line_number":194,"context_line":"multipath for the volume replica devices."}],"source_content_type":"text/x-rst","patch_set":4,"id":"227f67ee_4cb95008","line":191,"range":{"start_line":172,"start_character":0,"end_line":191,"end_character":5},"updated":"2021-12-02 12:05:18.000000000","message":"-1: This may be misleading for people that are not all that familiar with the attach flow, after all just because the Cinder driver returns a connection information dictionary that supports native NVMeOF multipathing doesn\u0027t mean that the connection (os-brick method connect_volume) will be done using multipathing, as this depends of the \"use_multipath\" parameter that was used to instantiate the `NVMeOFConnector` connector class.\n\nI think we need to rephrase the \"Single pathed connection\" and the \"Native multipathed connection\" to something like \"Connection information for cinder driver that only supports single path connections\" and \"Connection information for cinder driver that supports native multipathed connections (os-brick will iterate them if a single path connection is requested)\".\n\nWe should remind that just because the backend supports native multipathing doesn\u0027t mean that it will be used, since the connecting host must also support it.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":169,"context_line":"- RAID connection (single or multi pathed)"},{"line_number":170,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":171,"context_line":""},{"line_number":172,"context_line":"Single pathed connection:"},{"line_number":173,"context_line":"  .. code-block:: python"},{"line_number":174,"context_line":""},{"line_number":175,"context_line":"    {"},{"line_number":176,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":177,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":178,"context_line":"        \u0027portals\u0027: [(\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":179,"context_line":"    }"},{"line_number":180,"context_line":""},{"line_number":181,"context_line":"Native multipathed connection:"},{"line_number":182,"context_line":"  .. code-block:: python"},{"line_number":183,"context_line":""},{"line_number":184,"context_line":"    {"},{"line_number":185,"context_line":"        \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":186,"context_line":"        \u0027target_nqn\u0027: \u0027...somenqn...\u0027,"},{"line_number":187,"context_line":"        \u0027portals\u0027: ["},{"line_number":188,"context_line":"            (\u002710.1.1.1\u0027, 4420, \u0027tcp\u0027),"},{"line_number":189,"context_line":"            (\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)"},{"line_number":190,"context_line":"        ]"},{"line_number":191,"context_line":"    }"},{"line_number":192,"context_line":""},{"line_number":193,"context_line":"For RAID and DeviceMapper, the example below is given with no native"},{"line_number":194,"context_line":"multipath for the volume replica devices."}],"source_content_type":"text/x-rst","patch_set":4,"id":"57998535_96e142e2","line":191,"range":{"start_line":172,"start_character":0,"end_line":191,"end_character":5},"in_reply_to":"227f67ee_4cb95008","updated":"2021-12-14 06:13:52.000000000","message":"Ok, I think I understand what you are asking, I simply renamed the titles of the code blocks as you suggested (except removed the \"only supports single path...\" from the single path connection title - because driver doesnt only have to support it to work with single path) marking this resolved for now, please comment if more changes needed.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":195,"context_line":"For these replica devices to do native multipathing, portals for each of"},{"line_number":196,"context_line":"each device\u0027s paths should be specified in the `portals` list."},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":199,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":200,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replica"},{"line_number":201,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DeviceMapper"},{"line_number":202,"context_line":"and the replica devices are left hanging, how would we recognize this? And"},{"line_number":203,"context_line":"would we want to fail back to using RAID here instead?)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"57795e18_98e537cb","line":200,"range":{"start_line":198,"start_character":0,"end_line":200,"end_character":53},"updated":"2021-12-02 12:05:18.000000000","message":"-1: The spec cannot leave this open to the implementation, it must specify what we want to do.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":195,"context_line":"For these replica devices to do native multipathing, portals for each of"},{"line_number":196,"context_line":"each device\u0027s paths should be specified in the `portals` list."},{"line_number":197,"context_line":""},{"line_number":198,"context_line":"For RAID, the below example would have a `use_raid` flag added, unless there"},{"line_number":199,"context_line":"will be a different way for determining use of RAID vs DeviceMapper"},{"line_number":200,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replica"},{"line_number":201,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DeviceMapper"},{"line_number":202,"context_line":"and the replica devices are left hanging, how would we recognize this? And"},{"line_number":203,"context_line":"would we want to fail back to using RAID here instead?)"}],"source_content_type":"text/x-rst","patch_set":4,"id":"db48b4bd_f349a569","line":200,"range":{"start_line":198,"start_character":0,"end_line":200,"end_character":53},"in_reply_to":"57795e18_98e537cb","updated":"2021-12-14 06:13:52.000000000","message":"resolved by changing the DM replicas list field name","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":200,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replica"},{"line_number":201,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DeviceMapper"},{"line_number":202,"context_line":"and the replica devices are left hanging, how would we recognize this? And"},{"line_number":203,"context_line":"would we want to fail back to using RAID here instead?)"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"DeviceMapper connection:"},{"line_number":206,"context_line":"  .. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":4,"id":"0c079c4d_1dcbd804","line":203,"updated":"2021-12-02 12:05:18.000000000","message":"I think that it would be possible for the os-brick connector to do this automagically.\n\nWhen connecting to devices:\n\n- Devices with the same (NQN, namespace) where the client and backend supports ANA will be grouped automatically by the NVMe client.\n- Devices that are not grouped automatically but are the same device (because they are physically the same or they are replicated on the backend and presented as the same) will have the same WWID and will be grouped by multipathd.\n- All other devices (they don\u0027t have the same WWID) would need to be managed by the RAID.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":200,"context_line":"(consider a case where DeviceMapper usage is intended, but the volume replica"},{"line_number":201,"context_line":"devices don\u0027t have matching udev info, which does not trigger the DeviceMapper"},{"line_number":202,"context_line":"and the replica devices are left hanging, how would we recognize this? And"},{"line_number":203,"context_line":"would we want to fail back to using RAID here instead?)"},{"line_number":204,"context_line":""},{"line_number":205,"context_line":"DeviceMapper connection:"},{"line_number":206,"context_line":"  .. code-block:: python"}],"source_content_type":"text/x-rst","patch_set":4,"id":"cd7a1617_13009186","line":203,"in_reply_to":"0c079c4d_1dcbd804","updated":"2021-12-14 06:13:52.000000000","message":"That is my thinking too, I like this. I think we should be clear on this with this latest revision.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":219,"context_line":"                \u0027portals\u0027: [(\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":220,"context_line":"            }"},{"line_number":221,"context_line":"        ]"},{"line_number":222,"context_line":"    }"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":4,"id":"02e3113c_98e5ca0c","line":222,"updated":"2021-12-02 12:05:18.000000000","message":"Maybe we should add an additional example for the ANA + multipath case","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":true,"context_lines":[{"line_number":219,"context_line":"                \u0027portals\u0027: [(\u002710.1.1.2\u0027, 4420, \u0027tcp\u0027)]"},{"line_number":220,"context_line":"            }"},{"line_number":221,"context_line":"        ]"},{"line_number":222,"context_line":"    }"},{"line_number":223,"context_line":""},{"line_number":224,"context_line":""},{"line_number":225,"context_line":"Alternatives"}],"source_content_type":"text/x-rst","patch_set":4,"id":"2887be50_8e2b349c","line":222,"in_reply_to":"02e3113c_98e5ca0c","updated":"2021-12-14 06:13:52.000000000","message":"Intention here is same as described in comment on L170\n\nThe paragraph above (especially now with latest revision) says:\nFor DM, the example below is given with no native multipath for the replica devices.\nFor these replica devices to do native multipathing, portals for each of each device\u0027s paths should be specified in the `portals` list.\n\nAs in L170 - let me know if you think it would be better to expand both cases more explicitly here.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"0768e180c75535b2bac096a391fd5e15b0e4fe19","unresolved":true,"context_lines":[{"line_number":273,"context_line":""},{"line_number":274,"context_line":"Depending on the mode of multipath connection (for example active-active vs"},{"line_number":275,"context_line":"active-passive) - there could be an increase in network resource usage for"},{"line_number":276,"context_line":"the multipath device io."},{"line_number":277,"context_line":""},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"1a159a9f_ca64fd74","line":276,"updated":"2021-12-02 12:05:18.000000000","message":"We should also mention the potential performance benefit of higher I/O for the VMs, not only the downsides.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":273,"context_line":""},{"line_number":274,"context_line":"Depending on the mode of multipath connection (for example active-active vs"},{"line_number":275,"context_line":"active-passive) - there could be an increase in network resource usage for"},{"line_number":276,"context_line":"the multipath device io."},{"line_number":277,"context_line":""},{"line_number":278,"context_line":""},{"line_number":279,"context_line":"Other deployer impact"}],"source_content_type":"text/x-rst","patch_set":4,"id":"9daf886c_7a98339d","line":276,"in_reply_to":"1a159a9f_ca64fd74","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":20813,"name":"Sofia Enriquez","email":"lsofia.enriquez@gmail.com","username":"enriquetaso"},"change_message_id":"f8c25b7a6b256f889c5bab3fc9c4964b55f68c82","unresolved":true,"context_lines":[{"line_number":308,"context_line":""},{"line_number":309,"context_line":"Phase 1 (Native):"},{"line_number":310,"context_line":""},{"line_number":311,"context_line":"Connector properties check kernel supports native multipath"},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"Establish all (multiple) necessary connections to multipath volume and ensure"},{"line_number":314,"context_line":"proper device is exposed."},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Handle fallback to single pathed connections if enforce_multipath \u003d\u003d False"},{"line_number":317,"context_line":""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Phase 2 (DeviceMapper):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"91a16da0_19209e9f","line":316,"range":{"start_line":311,"start_character":0,"end_line":316,"end_character":74},"updated":"2021-11-15 21:21:12.000000000","message":":nit: maybe we can use bullets here to list the work items but I\u0027m only suggesting this for esthetic propose.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":308,"context_line":""},{"line_number":309,"context_line":"Phase 1 (Native):"},{"line_number":310,"context_line":""},{"line_number":311,"context_line":"Connector properties check kernel supports native multipath"},{"line_number":312,"context_line":""},{"line_number":313,"context_line":"Establish all (multiple) necessary connections to multipath volume and ensure"},{"line_number":314,"context_line":"proper device is exposed."},{"line_number":315,"context_line":""},{"line_number":316,"context_line":"Handle fallback to single pathed connections if enforce_multipath \u003d\u003d False"},{"line_number":317,"context_line":""},{"line_number":318,"context_line":""},{"line_number":319,"context_line":"Phase 2 (DeviceMapper):"}],"source_content_type":"text/x-rst","patch_set":4,"id":"df14f9a2_909d0912","line":316,"range":{"start_line":311,"start_character":0,"end_line":316,"end_character":74},"in_reply_to":"91a16da0_19209e9f","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"293a868d87332747700c4c88910ab2e043068a61","unresolved":true,"context_lines":[{"line_number":357,"context_line":"References"},{"line_number":358,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"  .. [1] ANA Spec - Section 8.1.3: https://nvmexpress.org/wp-content/uploads/NVMe-NVM-Express-2.0a-2021.07.26-Ratified.pdf"}],"source_content_type":"text/x-rst","patch_set":4,"id":"33950b72_be610ca4","line":360,"range":{"start_line":360,"start_character":0,"end_line":360,"end_character":2},"updated":"2021-11-10 13:34:44.000000000","message":"This extra space is causing this to be rendered as a blockquote, which is not what we want here.","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"9b2054353978de8c55e684d50006a8c834d4e193","unresolved":false,"context_lines":[{"line_number":357,"context_line":"References"},{"line_number":358,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":359,"context_line":""},{"line_number":360,"context_line":"  .. [1] ANA Spec - Section 8.1.3: https://nvmexpress.org/wp-content/uploads/NVMe-NVM-Express-2.0a-2021.07.26-Ratified.pdf"}],"source_content_type":"text/x-rst","patch_set":4,"id":"7fd365ec_050fa14a","line":360,"range":{"start_line":360,"start_character":0,"end_line":360,"end_character":2},"in_reply_to":"33950b72_be610ca4","updated":"2021-12-14 06:13:52.000000000","message":"Done","commit_id":"7aecef8dd1579f4ba1b6eea4e6b53b3cee40ad1b"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":15,"context_line":"* Native NVMeoF multipathing"},{"line_number":16,"context_line":"* DeviceMapper based multipathing"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"At the time of writing - only TCP and RDMA protocols are supported and"},{"line_number":19,"context_line":"contemplated by the spec (so it does not contemplate the case for other"},{"line_number":20,"context_line":"transport protocols such as FC)"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"126c2bbe_1d7318d7","line":18,"range":{"start_line":18,"start_character":22,"end_line":18,"end_character":25},"updated":"2021-12-20 17:07:47.000000000","message":"nit: comma instead of dash","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[{"line_number":15,"context_line":"* Native NVMeoF multipathing"},{"line_number":16,"context_line":"* DeviceMapper based multipathing"},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"At the time of writing - only TCP and RDMA protocols are supported and"},{"line_number":19,"context_line":"contemplated by the spec (so it does not contemplate the case for other"},{"line_number":20,"context_line":"transport protocols such as FC)"},{"line_number":21,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"8c6bf757_22ae828b","line":18,"range":{"start_line":18,"start_character":22,"end_line":18,"end_character":25},"in_reply_to":"126c2bbe_1d7318d7","updated":"2021-12-21 11:43:52.000000000","message":"Done","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"At the time of writing - only TCP and RDMA protocols are supported and"},{"line_number":19,"context_line":"contemplated by the spec (so it does not contemplate the case for other"},{"line_number":20,"context_line":"transport protocols such as FC)"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"cfd1eea6_bd9ed15b","line":20,"updated":"2021-12-20 17:07:47.000000000","message":"nit: Missing the full stop at the end.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[{"line_number":17,"context_line":""},{"line_number":18,"context_line":"At the time of writing - only TCP and RDMA protocols are supported and"},{"line_number":19,"context_line":"contemplated by the spec (so it does not contemplate the case for other"},{"line_number":20,"context_line":"transport protocols such as FC)"},{"line_number":21,"context_line":""},{"line_number":22,"context_line":""},{"line_number":23,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":7,"id":"f7ef9649_77906620","line":20,"in_reply_to":"cfd1eea6_bd9ed15b","updated":"2021-12-21 11:43:52.000000000","message":"Done","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":7198,"name":"Jay Bryant","email":"jungleboyj@electronicjungle.net","username":"jsbryant"},"change_message_id":"76627c7cd45a828f8b0e62a383226acbc0ba45e6","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"e473ec3a_7aa942cd","line":114,"updated":"2021-12-15 19:25:35.000000000","message":"What happens in the case where the host doesn\u0027t support multipathing and enforce is set to true?","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"bdbe60e3_8932952a","line":114,"in_reply_to":"0e3e8185_b113aee9","updated":"2021-12-21 11:43:52.000000000","message":"I think this spec should avoid changing in any code patterns outside the connector and driver connection_info API.\n\nAnd given the two very dangerous looking scenarios you described, we may simply want to avoid enforce_multipath for native multipath altogether (so native multipathing should assume enforcement of multipath and fail otherwise - to be safe rather than sorry)\nIf I understand correctly, these issues will not happen with DM multipath, (is that right?) so for DM we could do it the way os-brick already does it with other protocols. I could use some more input here.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"0e3e8185_b113aee9","line":114,"in_reply_to":"79663fea_4b48074d","updated":"2021-12-20 17:07:47.000000000","message":"We may have a problem here, because the enforce multipathing check is something that currently happens on the host when it calls the \"get_connector_properties\" before calling cinder to get the connection information.\n\nChanging it to be a driver responsibility would be a change in our code pattern and it may not solve all possible issues.\n\nAs far as I see it we have a couple of problems with using the enforce multipathing parameter, because Nova has a single parameter for this configuration that will be used for all os-brick calls, so we\u0027ll have problems if:\n\n- We only have NVMe volumes and we want to only do native multipathing:  \"get_connector_properties\" will fail if enforce multipathing is True and multipathd is not running, even if we don\u0027t want to use it.\n\n- We change the NVMe connector code to fail on the \"get_connector_properties\" if enforce multipathing is set to True and the host doesn\u0027t support native multipathing.  Doing this could break deployments that currently don\u0027t use NVMe devices and don\u0027t have native multipathing enabled/supported.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"8d6c7cba5aa6f8fad95f2428d7dfb4598752d944","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"80735381_0cba1f9b","line":114,"in_reply_to":"7c1d2a84_6c4153a1","updated":"2021-12-22 04:21:00.000000000","message":"Right, I see the point now, I was misunderstanding enforce_multipath.\n\nMy thinking was among the lines of: if driver wants to do native multipathing (knowing host does support it) - but for some reason host fails to execute it, \"enforce\u003dTrue\" would mean fail operation (or switch to DM) - rather than fallback to singlepath.\n\nBut now I\u0027m understanding the point of enforce_multipath is different than what I imagined, it is not enforcement based on host failure to execute intended multipathed connection, but based on its capabilities (a check if multipathd is running)\n\nI think this is cleared up for me now (unless im still misunderstanding something major here? 😊)\n\nAdding in your suggestion that enforce_multipath is a DM parameter and will not have interaction with native NVMe multipathing.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"7aad36d1010e20896458fe346285d62b475d58d1","unresolved":false,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"0af9b149_42669614","line":114,"in_reply_to":"80735381_0cba1f9b","updated":"2021-12-23 10:14:28.000000000","message":"Yes, that is the current behavior of the enforce multipath.  Which I don\u0027t think is reasonable, as it should be what you were assuming before, attachments should fail if we are not able to get a multipath.\n\nThe problem with changing this behavior now is that it would be kind of a breaking change. :-(","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"da5a46870f2f74ddfe03d813b200f0b2824d49d5","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"7c1d2a84_6c4153a1","line":114,"in_reply_to":"bdbe60e3_8932952a","updated":"2021-12-21 15:49:05.000000000","message":"I think we can just say in the spec that enforce_multipath is a DM multpathing parameter and it will not have any interaction with the native NVMe multipathing.\n\nI don\u0027t understand what you mean by assume enforce multipathing for native.  The cinder driver will know if the host that wants to connect to the volume supports native or not, so if it doesn\u0027t support it and wants to do multipathing it will have to do it with the DM, right?","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"4537fe336cb66c8b84b200bd8a8fc5a0a3cc9b6d","unresolved":true,"context_lines":[{"line_number":111,"context_line":"  }"},{"line_number":112,"context_line":""},{"line_number":113,"context_line":"In the case where the host does not support multipathing and the"},{"line_number":114,"context_line":"`enforce_multipath` flag is set to False, then fallback to the original single"},{"line_number":115,"context_line":"path behavior where only the first connection is established and that device"},{"line_number":116,"context_line":"is exposed."},{"line_number":117,"context_line":""}],"source_content_type":"text/x-rst","patch_set":7,"id":"79663fea_4b48074d","line":114,"in_reply_to":"e473ec3a_7aa942cd","updated":"2021-12-16 10:51:13.000000000","message":"That should cause an error.\n\nThis should happen on the cinder driver level: since the driver knows what kind of connection it wants to make (multipath mechanism and enforcement) - and it knows the host\u0027s connector properties (which will tell it if the host supports multipathing)","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"05527ac16b17b1329c1f81298016546f9b825ff4","unresolved":true,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"  {"},{"line_number":147,"context_line":"      \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":148,"context_line":"      \u0027dm_replicas\u0027: ["},{"line_number":149,"context_line":"          {"},{"line_number":150,"context_line":"              \u0027vol_uuid\u0027: \u0027...someuuid1...\u0027,"},{"line_number":151,"context_line":"              \u0027target_nqn\u0027: \u0027...somenqn1...\u0027,"}],"source_content_type":"text/x-rst","patch_set":7,"id":"e81f1ec6_13dc215f","line":148,"range":{"start_line":148,"start_character":7,"end_line":148,"end_character":18},"updated":"2021-12-20 17:07:47.000000000","message":"How does this interact with the existing \"volume_replicas\" key?\nIs it one or the other?\nWe should say it somewhere","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":16721,"name":"Zohar Mamedov","email":"zohar.cloud@gmail.com","username":"zohar"},"change_message_id":"a5d9da59a291dd5730835a65bf584ec883de2b6e","unresolved":false,"context_lines":[{"line_number":145,"context_line":""},{"line_number":146,"context_line":"  {"},{"line_number":147,"context_line":"      \u0027vol_uuid\u0027: \u0027...someuuid...\u0027,"},{"line_number":148,"context_line":"      \u0027dm_replicas\u0027: ["},{"line_number":149,"context_line":"          {"},{"line_number":150,"context_line":"              \u0027vol_uuid\u0027: \u0027...someuuid1...\u0027,"},{"line_number":151,"context_line":"              \u0027target_nqn\u0027: \u0027...somenqn1...\u0027,"}],"source_content_type":"text/x-rst","patch_set":7,"id":"d59c2433_23d2a9de","line":148,"range":{"start_line":148,"start_character":7,"end_line":148,"end_character":18},"in_reply_to":"e81f1ec6_13dc215f","updated":"2021-12-21 11:43:52.000000000","message":"It should be one or the other.\n\nAdding it to the top of this section.","commit_id":"f6ab8a0a767d31d78db229b7c541b331c4ce7099"},{"author":{"_account_id":9535,"name":"Gorka Eguileor","email":"geguileo@redhat.com","username":"Gorka"},"change_message_id":"da5a46870f2f74ddfe03d813b200f0b2824d49d5","unresolved":false,"context_lines":[{"line_number":174,"context_line":"- RAID connection (single or multipathed)"},{"line_number":175,"context_line":"- DeviceMapper connection (single or native multipath)"},{"line_number":176,"context_line":""},{"line_number":177,"context_line":"Note: RAID on top of DM is not allowed as it is inconsistent with the"},{"line_number":178,"context_line":"connection information API (and would need an unnecessary overcomplication of"},{"line_number":179,"context_line":"it as well as the physical implementation). It could theoretically be added"},{"line_number":180,"context_line":"one day based on need, but it is overkill on multiple levels."},{"line_number":181,"context_line":""},{"line_number":182,"context_line":"Single path connection information for cinder driver:"},{"line_number":183,"context_line":""}],"source_content_type":"text/x-rst","patch_set":8,"id":"84e4f0ee_4548461e","line":180,"range":{"start_line":177,"start_character":0,"end_line":180,"end_character":61},"updated":"2021-12-21 15:49:05.000000000","message":"nice!","commit_id":"cc3a24cca4e9ff02f10d3917d2314e29081345d1"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"ab5273465e5be58b64ed7851a93629185cc491da","unresolved":true,"context_lines":[{"line_number":116,"context_line":"Phase 2 (DeviceMapper)"},{"line_number":117,"context_line":"----------------------"},{"line_number":118,"context_line":""},{"line_number":119,"context_line":"For DeviceMapper multipathing, use `dm_replicas` in the connection information"},{"line_number":120,"context_line":"to specify all the replicas. This approach allows for multiple NQNs for one"},{"line_number":121,"context_line":"synced volume, which is often a requirement for DM-based multipathing."},{"line_number":122,"context_line":""}],"source_content_type":"text/x-rst","patch_set":9,"id":"fedd9f82_21668002","line":119,"range":{"start_line":119,"start_character":35,"end_line":119,"end_character":36},"updated":"2021-12-22 16:44:06.000000000","message":"nit: don\u0027t fix this -- you\u0027ve used the single-backticks consistently.  Just for your future specs: single-backticks gets you italics; you need to use double-backticks to get the monospace font.  (I think single-backtick gives you monospace in Markdown.)","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"},{"author":{"_account_id":5314,"name":"Brian Rosmaita","email":"rosmaita.fossdev@gmail.com","username":"brian-rosmaita"},"change_message_id":"ab5273465e5be58b64ed7851a93629185cc491da","unresolved":true,"context_lines":[{"line_number":166,"context_line":"---------------------"},{"line_number":167,"context_line":""},{"line_number":168,"context_line":"With this proposal, there will be multiple replication and/or multipath mode"},{"line_number":169,"context_line":"permutations of operation for the connector:"},{"line_number":170,"context_line":"- Single pathed connection"},{"line_number":171,"context_line":"- Native multipathed connection"},{"line_number":172,"context_line":"- RAID connection (single or multipathed)"}],"source_content_type":"text/x-rst","patch_set":9,"id":"4afb3a41_e5b4fe02","line":169,"updated":"2021-12-22 16:44:06.000000000","message":"nit: only fix if you need to push a new patch -- you need a blank line before line 170 or else sphinx doesn\u0027t recognize that lines 170-173 are supposed to be a bulleted list.","commit_id":"6a3ab81771c1166abb552cd8e69460417462ad90"}]}