Re: BUG #17912: Invalid memory access when converting plpython' array containing empty array

PG Bug reporting form <noreply@postgresql.org> writes:
> CREATE EXTENSION plpython3u;
> CREATE OR REPLACE FUNCTION test() RETURNS text[] AS $$
> return [[], "a"]
> $$ LANGUAGE plpython3u;
> SELECT test();

> As I can see, for the first case we get len = 0 in PLySequence_ToArray();
> elems, nulls palloc'ed with zero elements, but PLyObject_ToScalar() tries
> to write a value into nulls[0]...

Yeah.  The calculation of the array size is being done in the wrong
place, so that we may update len to zero before breaking out of the
loop.  But really it's poor coding for this function to be doing
its own calculation of the array size at all, rather than consulting
the authoritative ArrayGetNItems().  I think we need something like
the attached.

            regards, tom lane

diff --git a/src/pl/plpython/plpy_typeio.c b/src/pl/plpython/plpy_typeio.c
index 7018c9d404..864b5f1765 100644
--- a/src/pl/plpython/plpy_typeio.c
+++ b/src/pl/plpython/plpy_typeio.c
@@ -1136,7 +1136,7 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
     int            i;
     Datum       *elems;
     bool       *nulls;
-    int64        len;
+    int            len;
     int            ndim;
     int            dims[MAXDIM];
     int            lbs[MAXDIM];
@@ -1155,7 +1155,6 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
      * Determine the number of dimensions, and their sizes.
      */
     ndim = 0;
-    len = 1;

     Py_INCREF(plrv);

@@ -1174,17 +1173,6 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
         if (dims[ndim] < 0)
             PLy_elog(ERROR, "could not determine sequence length for function return value");

-        if (dims[ndim] > MaxAllocSize)
-            ereport(ERROR,
-                    (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-                     errmsg("array size exceeds the maximum allowed")));
-
-        len *= dims[ndim];
-        if (len > MaxAllocSize)
-            ereport(ERROR,
-                    (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
-                     errmsg("array size exceeds the maximum allowed")));
-
         if (dims[ndim] == 0)
         {
             /* empty sequence */
@@ -1214,15 +1202,18 @@ PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
                      errmsg("return value of function with array return type is not a Python sequence")));

         ndim = 1;
-        len = dims[0] = PySequence_Length(plrv);
+        dims[0] = PySequence_Length(plrv);
     }

+    /* Allocate space for work arrays, after detecting array size overflow */
+    len = ArrayGetNItems(ndim, dims);
+    elems = palloc(sizeof(Datum) * len);
+    nulls = palloc(sizeof(bool) * len);
+
     /*
      * Traverse the Python lists, in depth-first order, and collect all the
      * elements at the bottom level into 'elems'/'nulls' arrays.
      */
-    elems = palloc(sizeof(Datum) * len);
-    nulls = palloc(sizeof(bool) * len);
     currelem = 0;
     PLySequence_ToArray_recurse(arg->u.array.elm, plrv,
                                 dims, ndim, 0,