Unexpected behaviour, definitely looks like a bug.

PostgresQL version 9.4.20

WITH test(id, not_id) AS (
  SELECT * FROM (VALUES (1,1), (3,4)) AS t
),

test2(id1) AS (
  SELECT * FROM (VALUES (2), (3) ) AS t
)
-- SELECT not_id FROM test2  -- ERROR column "not_id" does not exist

SELECT * FROM test WHERE id IN ( SELECT not_id FROM test2 ); -- <-- not_id selected from test table no ERROR, result is: 1,1