ENH: continuing #1349, optimize DataFrame.cov for missing data

Author: adamklein

pandas/core/frame.py

@@ -3838,27 +3838,16 @@ def cov(self):
         y : DataFrame
         """
         numeric_df = self._get_numeric_data()
-        mat = numeric_df.values.T
         cols = numeric_df.columns
-        baseCov = np.cov(mat)
+        mat = numeric_df.values
 
-        for i, j, ac, bc in self._cov_helper(mat):
-            c = np.cov(ac, bc)[0, 1]
-            baseCov[i, j] = c
-            baseCov[j, i] = c
+        if notnull(mat).all():
+            baseCov = np.cov(mat.T)
+        else:
+            baseCov = lib.nancorr(mat, cov=True)
 
         return self._constructor(baseCov, index=cols, columns=cols)
 
-    def _cov_helper(self, mat):
-        # Get the covariance with items that have NaN values
-        mask = np.isfinite(mat)
-        for i, A in enumerate(mat):
-            if not mask[i].all():
-                for j, B in enumerate(mat):
-                    in_common = mask[i] & mask[j]
-                    if in_common.any():
-                        yield i, j, A[in_common], B[in_common]
-
     def corrwith(self, other, axis=0, drop=False):
         """
         Compute pairwise correlation between rows or columns of two DataFrame

pandas/src/moments.pyx

@@ -248,11 +248,11 @@ def ewma(ndarray[double_t] input, double_t com):
     return output
 
 #----------------------------------------------------------------------
-# Pairwise covariance
+# Pairwise correlation/covariance
 
 @cython.boundscheck(False)
 @cython.wraparound(False)
-def nancorr(ndarray[float64_t, ndim=2] mat):
+def nancorr(ndarray[float64_t, ndim=2] mat, cov=False):
     cdef:
         Py_ssize_t i, j, xi, yi, N, K
         ndarray[float64_t, ndim=2] result
@@ -294,7 +294,7 @@ def nancorr(ndarray[float64_t, ndim=2] mat):
                         sumxx += vx * vx
                         sumyy += vy * vy
 
-                divisor = sqrt(sumxx * sumyy)
+                divisor = (nobs - 1.0) if cov else sqrt(sumxx * sumyy)
 
                 if divisor != 0:
                     result[xi, yi] = result[yi, xi] = sumx / divisor