mgr/nfs: Add parameter to nfs cluster create to pass ingress placement

Fixes: https://tracker.ceph.com/issues/73718

Signed-off-by: Shweta Bhosale <Shweta.Bhosale1@ibm.com>

doc/mgr/nfs.rst

@@ -31,7 +31,7 @@ Create NFS Ganesha Cluster
 
 .. prompt:: bash #
 
-   ceph nfs cluster create <cluster_id> [<placement>] [--ingress] [--virtual_ip <value>] [--ingress-mode {default|keepalive-only|haproxy-standard|haproxy-protocol}] [--port <int>] [--enable-rdma] [--rdma_port <int>] [-i <spec_file>] [--enable-nfsv3]
+   ceph nfs cluster create <cluster_id> [<placement>] [--ingress] [--virtual_ip <value>] [--ingress-mode {default|keepalive-only|haproxy-standard|haproxy-protocol}] [--ingress-placement <placement>] [--port <int>] [--enable-rdma] [--rdma_port <int>] [--enable-nfsv3] [-i <spec_file>]
 
 This creates a common recovery pool for all NFS Ganesha daemons, new user based on
 ``cluster_id``, and a common NFS Ganesha config RADOS object.
@@ -111,6 +111,18 @@ of the details of NFS redirecting traffic on the virtual IP to the
 appropriate backend NFS servers, and redeploying NFS servers when they
 fail.
 
+By default, the *ingress* service follows the same placement as the NFS
+Ganesha daemons (the optional ``<placement>`` argument).  To schedule
+keepalived and HAProxy on a different set of hosts, pass
+``--ingress-placement`` with a separate placement string.  For example,
+to run three NFS daemons on ``host1`` and ``host2`` while colocating
+ingress on three dedicated nodes::
+
+    ceph nfs cluster create mynfs "2 host1 host2" --ingress --virtual_ip 192.168.1.100/24 --ingress-placement "3 host3 host4 host5"
+
+If ``--ingress-placement`` is omitted, both services share the NFS
+placement.
+
 An optional ``--ingress-mode`` parameter can be provided to choose
 how the *ingress* service is configured:
 

src/pybind/mgr/nfs/cluster.py

@@ -172,6 +172,7 @@ class NFSCluster:
             tls_ciphers: Optional[str] = None,
             enable_rdma: bool = False,
             rdma_port: Optional[int] = None,
+            ingress_placement: Optional[str] = None,
     ) -> None:
         if not port:
             port = 2049   # default nfs port
@@ -182,9 +183,12 @@ class NFSCluster:
                 ingress_mode = IngressType.default
             ingress_mode = ingress_mode.canonicalize()
             pspec = PlacementSpec.from_string(placement)
+            ingress_pspec = PlacementSpec.from_string(ingress_placement) if ingress_placement else None
             if ingress_mode == IngressType.keepalive_only:
                 # enforce count=1 for nfs over keepalive only
                 pspec.count = 1
+                if ingress_pspec:
+                    ingress_pspec.count = 1
 
             ganesha_port = 10000 + port  # semi-arbitrary, fix me someday
             frontend_port: Optional[int] = port
@@ -223,7 +227,7 @@ class NFSCluster:
             ispec = IngressSpec(service_type='ingress',
                                 service_id='nfs.' + cluster_id,
                                 backend_service='nfs.' + cluster_id,
-                                placement=pspec,
+                                placement=ingress_pspec or pspec,
                                 frontend_port=frontend_port,
                                 monitor_port=7000 + port,   # semi-arbitrary, fix me someday
                                 virtual_ip=virtual_ip,
@@ -283,6 +287,7 @@ class NFSCluster:
             tls_ciphers: Optional[str] = None,
             enable_rdma: bool = False,
             rdma_port: Optional[int] = None,
+            ingress_placement: Optional[str] = None,
     ) -> None:
         try:
             if virtual_ip:
@@ -323,7 +328,8 @@ class NFSCluster:
                     tls_min_version=tls_min_version,
                     tls_ciphers=tls_ciphers,
                     enable_rdma=enable_rdma,
-                    rdma_port=rdma_port
+                    rdma_port=rdma_port,
+                    ingress_placement=ingress_placement
                 )
                 return
             raise NonFatalError(f"{cluster_id} cluster already exists")

src/pybind/mgr/nfs/module.py

@@ -165,6 +165,7 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                                 enable_rdma: bool = False,
                                 rdma_port: Optional[int] = None,
                                 enable_nfsv3: bool = False,
+                                ingress_placement: Optional[str] = None,
                                 inbuf: Optional[str] = None) -> None:
         """Create an NFS Cluster"""
         cluster_qos_config = None
@@ -196,7 +197,8 @@ class Module(orchestrator.OrchestratorClientMixin, MgrModule):
                                            tls_min_version=tls_min_version,
                                            tls_ciphers=tls_ciphers,
                                            enable_rdma=enable_rdma,
-                                           rdma_port=rdma_port)
+                                           rdma_port=rdma_port,
+                                           ingress_placement=ingress_placement)
 
     @NFSCLICommand('nfs cluster rm', perm='rw')
     @object_format.EmptyResponder()

src/pybind/mgr/nfs/tests/test_nfs.py

@@ -11,7 +11,7 @@ from mgr_module import MgrModule, NFS_POOL_NAME
 
 from rados import ObjectNotFound
 
-from ceph.deployment.service_spec import NFSServiceSpec
+from ceph.deployment.service_spec import NFSServiceSpec, PlacementSpec
 from ceph.utils import with_units_to_int, bytes_to_human
 from nfs import Module
 from nfs.export import ExportMgr, normalize_path
@@ -1798,6 +1798,30 @@ EXPORT {
         self._do_mock_test(self._do_test_export_qos_bw_ops, qos_type, clust_bw_params, clust_ops_params, export_bw_params, export_ops_params)
 
 
+class TestNFSClusterIngressPlacement:
+    cluster_id = 'mynfs'
+    virtual_ip = '192.168.1.100/24'
+    nfs_placement = '2 host1 host2'
+    ingress_placement = '3 host3 host4 host5'
+
+    def test_create_nfs_cluster_passes_ingress_placement(self):
+        mgr = MagicMock()
+        cluster = NFSCluster(mgr)
+        with mock.patch('nfs.cluster.create_ganesha_pool'), \
+                mock.patch.object(cluster, 'create_empty_rados_obj'), \
+                mock.patch('nfs.cluster.available_clusters', return_value=[]), \
+                mock.patch.object(cluster, '_call_orch_apply_nfs') as mock_apply:
+            cluster.create_nfs_cluster(
+                cluster_id=self.cluster_id,
+                placement='host1',
+                virtual_ip=self.virtual_ip,
+                ingress=True,
+                ingress_placement=self.ingress_placement,
+            )
+            mock_apply.assert_called_once()
+            assert mock_apply.call_args.kwargs['ingress_placement'] == self.ingress_placement
+
+
 @pytest.mark.parametrize(
     "path,expected",
     [