)]}'
{"specs/rocky/approved/mixed-cpu-policy-instance.rst":[{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"1b12138136dd6133f942d1effff31534062027ca","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This feature enables \"vCPUs with shared pCPUs backend\" and \"vCPUs with"},{"line_number":14,"context_line":"dedicated pCPUs backend\" to co-locate on the same VM instance at the same time"},{"line_number":15,"context_line":"by introducing ``hw:dedicated_vcpus`` as an additional option of"},{"line_number":16,"context_line":"``hw:cpu-policy\u003ddedicated`` in the flavor extra spec."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_63ce7fb5","line":15,"range":{"start_line":15,"start_character":17,"end_line":15,"end_character":35},"updated":"2018-02-19 08:39:55.000000000","message":"Afterthought: Is \"hw:dedicated_vcpus\" the best option to add? Personally, I like introducing \"hw:shared_vcpus\" is better. As a user, I have already specified \"hw:cpu-policy\u003ddedicated\", so it is strange to specify dedicated vcpus again. Specifying \"hw:shared_vcpus\" for \"cpu-policy\u003ddedicated\" case sounds more natural to me.","commit_id":"2cb5d2bbb20151da3c3491ebde712e26c49140c8"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"8c70a19840da8b0f660ce7694ace9380aca9ceba","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This feature enables \"vCPUs with shared pCPUs backend\" and \"vCPUs with"},{"line_number":14,"context_line":"dedicated pCPUs backend\" to co-locate on the same VM instance at the same time"},{"line_number":15,"context_line":"by introducing ``hw:dedicated_vcpus`` as an additional option of"},{"line_number":16,"context_line":"``hw:cpu-policy\u003ddedicated`` in the flavor extra spec."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_1cf4b886","line":15,"range":{"start_line":15,"start_character":17,"end_line":15,"end_character":35},"in_reply_to":"3fa0c359_23c6374b","updated":"2018-02-19 11:48:16.000000000","message":"Done","commit_id":"2cb5d2bbb20151da3c3491ebde712e26c49140c8"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"7345c9c3f209747c242d7491825b37225807dd85","unresolved":false,"context_lines":[{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This feature enables \"vCPUs with shared pCPUs backend\" and \"vCPUs with"},{"line_number":14,"context_line":"dedicated pCPUs backend\" to co-locate on the same VM instance at the same time"},{"line_number":15,"context_line":"by introducing ``hw:dedicated_vcpus`` as an additional option of"},{"line_number":16,"context_line":"``hw:cpu-policy\u003ddedicated`` in the flavor extra spec."},{"line_number":17,"context_line":""},{"line_number":18,"context_line":"Problem description"}],"source_content_type":"text/x-rst","patch_set":1,"id":"3fa0c359_23c6374b","line":15,"range":{"start_line":15,"start_character":17,"end_line":15,"end_character":35},"in_reply_to":"3fa0c359_63ce7fb5","updated":"2018-02-19 09:04:44.000000000","message":"In other words,\n* Not specifying shared vcpus means no shared vcpu. \u003c- Cool !\n* Not specifying dedicated vcpus means all dedicated vcpu. \u003c- Hmm ?","commit_id":"2cb5d2bbb20151da3c3491ebde712e26c49140c8"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"fca88ca62b239a8a5f0dc4748fa0789ec5f95610","unresolved":false,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/mixed-cpu-policy-instance"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This feature enables \"vCPUs with shared pCPUs backend\" and \"vCPUs with"},{"line_number":14,"context_line":"dedicated pCPUs backend\" to co-locate on the same VM instance at the same time"},{"line_number":15,"context_line":"by introducing ``hw:shared_vcpus`` as an additional option of"},{"line_number":16,"context_line":"``hw:cpu-policy\u003ddedicated`` in the flavor extra spec."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_53108d59","line":14,"range":{"start_line":14,"start_character":25,"end_line":14,"end_character":78},"updated":"2018-02-22 18:58:59.000000000","message":"colocate isn\u0027t really the right word here... I think it\u0027s better to say\n\n\"This feature enables instances that have vCPUs that are pinned to specific pCPUs and vCPUs that are *not* pinned to specific pCPUs\".","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"f7cadadf9c72a154e8fc408ad4ec68feee354300","unresolved":false,"context_lines":[{"line_number":11,"context_line":"https://blueprints.launchpad.net/nova/+spec/mixed-cpu-policy-instance"},{"line_number":12,"context_line":""},{"line_number":13,"context_line":"This feature enables \"vCPUs with shared pCPUs backend\" and \"vCPUs with"},{"line_number":14,"context_line":"dedicated pCPUs backend\" to co-locate on the same VM instance at the same time"},{"line_number":15,"context_line":"by introducing ``hw:shared_vcpus`` as an additional option of"},{"line_number":16,"context_line":"``hw:cpu-policy\u003ddedicated`` in the flavor extra spec."},{"line_number":17,"context_line":""}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_be73d1e6","line":14,"range":{"start_line":14,"start_character":25,"end_line":14,"end_character":78},"in_reply_to":"3fa0c359_53108d59","updated":"2018-02-23 05:28:00.000000000","message":"Thank you for correcting. Done.","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"fca88ca62b239a8a5f0dc4748fa0789ec5f95610","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Some deployments may need to co-locate applications with high performance"},{"line_number":22,"context_line":"requirements alongside applications with no performance requirements in the"},{"line_number":23,"context_line":"same virtual machine. While high performance loads should be pinned to"},{"line_number":24,"context_line":"dedicated pCPUs, low performance loads are allowed to float on host pCPUs"},{"line_number":25,"context_line":"and to be shared with other processes for better utilization of host CPU"},{"line_number":26,"context_line":"resources."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_8e76eea3","line":23,"range":{"start_line":21,"start_character":0,"end_line":23,"end_character":21},"updated":"2018-02-22 18:58:59.000000000","message":"Again, colocate doesn\u0027t seem right the right word here. Perhaps better to reword this as:\n\n\"Some applications -- especially a subset of VNF applications -- require part of the application to run with guaranteed performance and other parts of the application do not have these requirements. Therefore, it would be beneficial to allow the VMs running these applications to pin *some* of the vCPUs to underlying pCPUs but allow other vCPUs to float over non-dedicated pCPUs.\"","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"f7cadadf9c72a154e8fc408ad4ec68feee354300","unresolved":false,"context_lines":[{"line_number":18,"context_line":"Problem description"},{"line_number":19,"context_line":"\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d\u003d"},{"line_number":20,"context_line":""},{"line_number":21,"context_line":"Some deployments may need to co-locate applications with high performance"},{"line_number":22,"context_line":"requirements alongside applications with no performance requirements in the"},{"line_number":23,"context_line":"same virtual machine. While high performance loads should be pinned to"},{"line_number":24,"context_line":"dedicated pCPUs, low performance loads are allowed to float on host pCPUs"},{"line_number":25,"context_line":"and to be shared with other processes for better utilization of host CPU"},{"line_number":26,"context_line":"resources."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_9eb37598","line":23,"range":{"start_line":21,"start_character":0,"end_line":23,"end_character":21},"in_reply_to":"3fa0c359_8e76eea3","updated":"2018-02-23 05:28:00.000000000","message":"Thank you so much. Done.","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":7730,"name":"Sahid Orentino Ferdjaoui","email":"sahid.ferdjaoui@industrialdiscipline.com","username":"sahid"},"change_message_id":"47ec1b08b019397afe787427f2d9177f75e48825","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Use Cases"},{"line_number":38,"context_line":"---------"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"As an end-user, I want vCPUs for processing network packets to be pinned to"},{"line_number":41,"context_line":"dedicated pCPUs to get maximum performance."},{"line_number":42,"context_line":"As an operator, I want to save compute resources for guest vCPUs which don’t"},{"line_number":43,"context_line":"require high performance."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_491f8807","line":40,"updated":"2018-02-22 15:04:01.000000000","message":"Not really sure of the use-case here, this can be achieve by launching an instance using dedicated policy and so running the high-performance process and an other-one with shared vCPUs. Are you sure you want to add such complexity?","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":7,"name":"Jay Pipes","email":"jaypipes@gmail.com","username":"jaypipes"},"change_message_id":"fca88ca62b239a8a5f0dc4748fa0789ec5f95610","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Use Cases"},{"line_number":38,"context_line":"---------"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"As an end-user, I want vCPUs for processing network packets to be pinned to"},{"line_number":41,"context_line":"dedicated pCPUs to get maximum performance."},{"line_number":42,"context_line":"As an operator, I want to save compute resources for guest vCPUs which don’t"},{"line_number":43,"context_line":"require high performance."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_53372dff","line":40,"in_reply_to":"3fa0c359_491f8807","updated":"2018-02-22 18:58:59.000000000","message":"Right. That\u0027s exactly what I said in revision 1 of the original spec :)\n\nhttps://review.openstack.org/#/c/543805/1/specs/rocky/approved/add-cpu-policy-mixed.rst@152","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"f7cadadf9c72a154e8fc408ad4ec68feee354300","unresolved":false,"context_lines":[{"line_number":37,"context_line":"Use Cases"},{"line_number":38,"context_line":"---------"},{"line_number":39,"context_line":""},{"line_number":40,"context_line":"As an end-user, I want vCPUs for processing network packets to be pinned to"},{"line_number":41,"context_line":"dedicated pCPUs to get maximum performance."},{"line_number":42,"context_line":"As an operator, I want to save compute resources for guest vCPUs which don’t"},{"line_number":43,"context_line":"require high performance."}],"source_content_type":"text/x-rst","patch_set":3,"id":"3fa0c359_de433d6b","line":40,"in_reply_to":"3fa0c359_53372dff","updated":"2018-02-23 05:28:00.000000000","message":"\u003e Right. That\u0027s exactly what I said in revision 1 of the original spec :)\n\nYup, I have put that proposal into alternative section. But we shouldn\u0027t give up trying to support the architecture as long as it is possible.\n\n\u003e Are you sure you want to add such complexity?\n\nGiven that the related spec supporting both VMs on one *host* (call it spec1 and this is spec2) is implemented and once shared pCPUs and dedicated pCPUs are both in NUMACell objects, I think this spec2 can be done without bringing a lot of complexity or breaking today\u0027s schema of cpu pinning.\n\nThe most difficult thing, I think, is how to get consistent with schedulers in spec1. But considering the situation now we are managing dedicated pCPUs and vCPUs in all of placement, requestspec, corefilter, numa object, and numatopologyfilter, I’m proposing to make it *simpler* by leaving them only to numa objects and numatopologyfilter.\n\nAnd thanks to the added shared pCPUs on the host, we might be able to let emulator threads always float there, and there might be no need to pinning them to dedicated pCPUs. Here, I’m not talking about the core utilization, I’m talking about the code maintenability (getting the code simpler).\n\nI believe the spec1 will lead us in the right direction to provide a simpler way to maximize performance and resource optimization with less LOC. And with that spec1, is this spec2 so complicated?","commit_id":"f00ad5f08bdd52e10f68435c005e596d2865c18c"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"975890c50f197771eb1af0301fae5b02ef2a718a","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"We introduce a new option ``hw:shared_vcpus`` in the flavor extra specs."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* ``hw:shared_vcpus\u003d\u003cvcpu number list\u003e``"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The ``hw:shared_vcpus`` field is only valid when the ``hw:cpu_policy`` is"},{"line_number":60,"context_line":"set to ``dedicated``. If the ``hw:shared_vcpus`` isn\u0027t set then the vCPUs"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_0e3f0087","line":57,"range":{"start_line":57,"start_character":1,"end_line":57,"end_character":40},"updated":"2018-03-16 17:42:36.000000000","message":"technically it\u0027s not a list, but a CPU set specification.  (I\u0027m assuming we\u0027d parse it with the existing parse_cpu_spec() routine.)","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"ab6206b6c14b5afa09069856d175e834a79578e0","unresolved":false,"context_lines":[{"line_number":54,"context_line":""},{"line_number":55,"context_line":"We introduce a new option ``hw:shared_vcpus`` in the flavor extra specs."},{"line_number":56,"context_line":""},{"line_number":57,"context_line":"* ``hw:shared_vcpus\u003d\u003cvcpu number list\u003e``"},{"line_number":58,"context_line":""},{"line_number":59,"context_line":"The ``hw:shared_vcpus`` field is only valid when the ``hw:cpu_policy`` is"},{"line_number":60,"context_line":"set to ``dedicated``. If the ``hw:shared_vcpus`` isn\u0027t set then the vCPUs"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_c88c9781","line":57,"range":{"start_line":57,"start_character":1,"end_line":57,"end_character":40},"in_reply_to":"df7087c5_0e3f0087","updated":"2018-03-17 02:33:48.000000000","message":"Done. (And I updated this part to use \"FLAVOR-CORES\" as well as in the user\u0027s flavor document)","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"975890c50f197771eb1af0301fae5b02ef2a718a","unresolved":false,"context_lines":[{"line_number":61,"context_line":"are all dedicated, if it is set then any vCPUs not listed are treated as"},{"line_number":62,"context_line":"dedicated vCPUs."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Every shared vCPU floats on the shared pCPUs in the host and every dedicated"},{"line_number":65,"context_line":"vCPU is pinned to one pCPU picked from dedicated pCPUs in the host."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Example"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_8ed8f01d","line":65,"range":{"start_line":64,"start_character":0,"end_line":65,"end_character":67},"updated":"2018-03-16 17:42:36.000000000","message":"Given the examples below, it looks like we won\u0027t let the \"shared\" vCPUs float across *all* the shared pCPUs on the host, but rather will restrict each to a single host NUMA node.\n\nWould we allow the libvirt driver to enforce that there must be at least one \"shared\" pCPU per host NUMA node?  (This would be ideal if we wanted to run the emulator threads on that \"shared\" pCPU.)","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"db9c58b6a036fff49c5cbde02f00e844581de824","unresolved":false,"context_lines":[{"line_number":61,"context_line":"are all dedicated, if it is set then any vCPUs not listed are treated as"},{"line_number":62,"context_line":"dedicated vCPUs."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Every shared vCPU floats on the shared pCPUs in the host and every dedicated"},{"line_number":65,"context_line":"vCPU is pinned to one pCPU picked from dedicated pCPUs in the host."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Example"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_a9f58985","line":65,"range":{"start_line":64,"start_character":0,"end_line":65,"end_character":67},"in_reply_to":"df7087c5_482ba72f","updated":"2018-03-19 15:57:44.000000000","message":"I like the idea of running the emulator threads on the \"shared\" pCPUs.  I think we might want to consider the numa topology though, since emulator and I/O threads can allocate memory.  On the other hand, maybe the added scheduling flexibility is worth the potential memory access latency.\n\nBut you\u0027re right that we should probably keep that for a follow-up spec.","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"ab6206b6c14b5afa09069856d175e834a79578e0","unresolved":false,"context_lines":[{"line_number":61,"context_line":"are all dedicated, if it is set then any vCPUs not listed are treated as"},{"line_number":62,"context_line":"dedicated vCPUs."},{"line_number":63,"context_line":""},{"line_number":64,"context_line":"Every shared vCPU floats on the shared pCPUs in the host and every dedicated"},{"line_number":65,"context_line":"vCPU is pinned to one pCPU picked from dedicated pCPUs in the host."},{"line_number":66,"context_line":""},{"line_number":67,"context_line":""},{"line_number":68,"context_line":"Example"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_482ba72f","line":65,"range":{"start_line":64,"start_character":0,"end_line":65,"end_character":67},"in_reply_to":"df7087c5_8ed8f01d","updated":"2018-03-17 02:33:48.000000000","message":"Good points. Answered inline.\n\n\u003e Given the examples below, it looks like we won\u0027t let the \"shared\" vCPUs float across *all* the shared pCPUs on the host.\n\nIf the instance has numa topology, yes because that is the current behavior of shared VMs. But if the instance doesn\u0027t have numa topology we will let the \"shared\" vCPUs float across *all* the shared pCPUs on the host because that is the current behavior of shared VMs. Added that case explicitly in the spec.\n\n\u003e Would we allow the libvirt driver to enforce that there must be at least one \"shared\" pCPU per host NUMA node?\n\nI\u0027d like to avoid to lay down additional limitation to deployers as much as possible. If there is no shared pCPU on a NUMA node of the host and the instance wants it then NUMATopologyFilter can exclude that host. That\u0027s how shared VM with numa topology is deployed today.\n\n\u003e This would be ideal if we wanted to run the emulator threads on that \"shared\" pCPU.\n\nI was thinking (out of the scope of this spec though) to run emulator threads on all the shared pCPUs on the hosts not considering NUMATopology. I don\u0027t see any problems here for today\u0027s requirement from NFV that \"we want to segregate emulator threads from dedicated cpus\", do we?","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":8768,"name":"Chris Friesen","email":"chris.friesen@windriver.com","username":"cbf123"},"change_message_id":"975890c50f197771eb1af0301fae5b02ef2a718a","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Example"},{"line_number":69,"context_line":"-------"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The compute node has 16 CPUs without hyper threading:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* NUMA node0 CPU(s): 0-7"},{"line_number":74,"context_line":"* NUMA node1 CPU(s): 8-15"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_ced10825","line":71,"range":{"start_line":71,"start_character":37,"end_line":71,"end_character":52},"updated":"2018-03-16 17:42:36.000000000","message":"\"hyperthreading\" or SMT","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"},{"author":{"_account_id":25625,"name":"Tetsuro Nakamura","email":"tetsuro.nakamura.bc@hco.ntt.co.jp","username":"tetsuro0907"},"change_message_id":"ab6206b6c14b5afa09069856d175e834a79578e0","unresolved":false,"context_lines":[{"line_number":68,"context_line":"Example"},{"line_number":69,"context_line":"-------"},{"line_number":70,"context_line":""},{"line_number":71,"context_line":"The compute node has 16 CPUs without hyper threading:"},{"line_number":72,"context_line":""},{"line_number":73,"context_line":"* NUMA node0 CPU(s): 0-7"},{"line_number":74,"context_line":"* NUMA node1 CPU(s): 8-15"}],"source_content_type":"text/x-rst","patch_set":9,"id":"df7087c5_e81f1b07","line":71,"range":{"start_line":71,"start_character":37,"end_line":71,"end_character":52},"in_reply_to":"df7087c5_ced10825","updated":"2018-03-17 02:33:48.000000000","message":"Done","commit_id":"fb906c9b0624a6186370144a66e42e51f20d4d1e"}]}