core: fix pw_core_find_format() for active ports

pw_core_find_format() is currently broken when one of the ports is already
active: The format of the active port is used and the other port is
completely ignored.
As a result, the autolink module may try to link a new port to the first
already active port even if the formats do not match.

To fix this, use the format of the active port as a filter and enumerate
the formats of the other port.

1 files changed, 27 insertions, 13 deletions

diff --git a/src/pipewire/core.c b/src/pipewire/core.c
index fcb362ca..23b3fba3 100644
--- a/src/pipewire/core.c
+++ b/src/pipewire/core.c
@@ -832,6 +832,9 @@ int pw_core_find_format(struct pw_core *core,
 	uint32_t out_state, in_state;
 	int res;
 	uint32_t iidx = 0, oidx = 0;
+	struct spa_pod_builder fb = { 0 };
+	uint8_t fbuf[4096];
+	struct spa_pod *filter;
 
 	out_state = output->state;
 	in_state = input->state;
@@ -848,36 +851,47 @@ int pw_core_find_format(struct pw_core *core,
 
 	if (in_state == PW_PORT_STATE_CONFIGURE && out_state > PW_PORT_STATE_CONFIGURE) {
 		/* only input needs format */
+		spa_pod_builder_init(&fb, fbuf, sizeof(fbuf));
 		if ((res = spa_node_port_enum_params_sync(output->node->node,
 						     output->direction, output->port_id,
 						     SPA_PARAM_Format, &oidx,
-						     NULL, format, builder)) != 1) {
-			if (res == 0)
-				res = -EBADF;
-			asprintf(error, "error get output format: %s", spa_strerror(res));
+						     NULL, &filter, &fb)) != 1) {
+			asprintf(error, "error get output format: %d", res);
 			goto error;
 		}
 		pw_log_debug("Got output format:");
 		if (pw_log_level_enabled(SPA_LOG_LEVEL_DEBUG))
-			spa_debug_format(2, NULL, *format);
+			spa_debug_format(2, NULL, filter);
+
+		if ((res = spa_node_port_enum_params_sync(input->node->node,
+						     input->direction, input->port_id,
+						     SPA_PARAM_EnumFormat, &iidx,
+						     filter, format, builder)) <= 0) {
+			asprintf(error, "error input enum formats: %d", res);
+			goto error;
+		}
 	} else if (out_state >= PW_PORT_STATE_CONFIGURE && in_state > PW_PORT_STATE_CONFIGURE) {
 		/* only output needs format */
+		spa_pod_builder_init(&fb, fbuf, sizeof(fbuf));
 		if ((res = spa_node_port_enum_params_sync(input->node->node,
 						     input->direction, input->port_id,
 						     SPA_PARAM_Format, &iidx,
-						     NULL, format, builder)) != 1) {
-			if (res == 0)
-				res = -EBADF;
-			asprintf(error, "error get input format: %s", spa_strerror(res));
+						     NULL, &filter, &fb)) != 1) {
+			asprintf(error, "error get input format: %d", res);
 			goto error;
 		}
 		pw_log_debug("Got input format:");
 		if (pw_log_level_enabled(SPA_LOG_LEVEL_DEBUG))
-			spa_debug_format(2, NULL, *format);
+			spa_debug_format(2, NULL, filter);
+
+		if ((res = spa_node_port_enum_params_sync(output->node->node,
+						     output->direction, output->port_id,
+						     SPA_PARAM_EnumFormat, &oidx,
+						     filter, format, builder)) <= 0) {
+			asprintf(error, "error output enum formats: %d", res);
+			goto error;
+		}
 	} else if (in_state == PW_PORT_STATE_CONFIGURE && out_state == PW_PORT_STATE_CONFIGURE) {
-		struct spa_pod_builder fb = { 0 };
-		uint8_t fbuf[4096];
-		struct spa_pod *filter;
 	      again:
 		/* both ports need a format */
 		pw_log_debug("core %p: do enum input %d", core, iidx);