From b69df42cf7c4b8e5fa95d38bd2c0f8ddeaa9223f Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Wed, 9 Dec 2020 10:04:54 -0500
Subject: [PATCH 01/20] ENH: start calculation using CMI PDF calculators.

Import PDFCalc and DebyeCalc from srreal to replace PdfFit calculation.
---
 src/diffpy/pdfgui/control/calculation.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index 4417be2b..34ae89c1 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -25,6 +25,8 @@
 from diffpy.pdfgui.control.pdfcomponent import PDFComponent
 from diffpy.pdfgui.utils import safeCPickleDumps, pickle_loads
 
+from diffpy.srreal.pdfcalculator import PDFCalculator, DebyePDFCalculator
+
 
 class Calculation(PDFComponent):
     """Perform a theoretical computation of PDF from model structure.
@@ -65,10 +67,12 @@ def __init__(self, name):
         self.stype = 'X'
         # user must specify qmax to get termination ripples
         self.qmax = 0.0
+        self.qmin = 0.0
         self.qdamp = 0.001
         self.qbroad = 0.0
         self.spdiameter = None
         self.dscale = 1.0
+        self.pctype = 'PC' # default use PDF real space calculator
         return
 
     def _getStrId(self):
@@ -150,6 +154,20 @@ def calculate(self):
         from diffpy.pdffit2 import PdfFit
         server = PdfFit()
 
+        if self.pctype == 'PC': # use PDFCalculator
+            pc = PDFCalculator()
+        elif self.pctype == 'DPC': # use DebyePDFCalculator
+            pc = DebyePDFCalculator()
+
+        pc.qmax = self.qmax
+        pc.qmin = self.qmin
+        pc.qdamp = self.qdamp
+        pc.qbroad = self.qbroad
+        pc.rmin = self.rmin
+        pc.rmax = self.rmax
+        pc.rstep = self.rstep
+        pc.scale = self.dscale
+
         # structure needs to be read before dataset allocation
         for struc in self.owner.strucs:
             server.read_struct_string(struc.writeStr('pdffit'))

From eddcb4326cf5b061d4616d8c67c6c1102dbe5517 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Wed, 6 Jan 2021 06:52:37 -0500
Subject: [PATCH 02/20] ENH: store calculation for multi-phases.

Use `pc_temp` for each phase, store the calculated gr to a list.
Sum the list to get the final calculation results.
---
 src/diffpy/pdfgui/control/calculation.py | 41 ++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index 34ae89c1..ef78ce97 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -168,6 +168,47 @@ def calculate(self):
         pc.rstep = self.rstep
         pc.scale = self.dscale
 
+        # load structure and disable metadata using the nometa function
+        # and set any calculator attributes as needed as above
+        r_list = []
+        g_list = []
+        for struc in self.owner.strucs:
+            # pc is for one calculation. the common setting
+            # pc_temp is for each phase, specific setting
+            pc_temp = pc.copy()
+            pc_temp.delta1 = struc.getvar('delta1')
+            pc_temp.delta2 = struc.getvar('delta2')
+            if struc.getvar('pscale'):
+                pc_temp.addEnvelope('scale')
+                pc_temp.scale = struc.getvar('pscale')
+            if struc.getvar('spdiameter'):
+                pc_temp.addEnvelope('sphericalshape')
+                pc_temp.spdiameter = struc.getvar('spdiameter')
+            if struc.getvar('stepcut'):
+                pc_temp.addEnvelope('stepcut')
+                pc_temp.stepcut = struc.getvar('stepcut')
+
+
+            ##TODO: pair mask. the baseline is not correct with PDFFIT.
+            pc_temp = struc.applyCMIPairSelection(pc_temp)
+            # pc_temp.setTypeMask("Ni","O",True)
+
+            r, g = pc_temp(nometa(struc))
+            g = g * struc.getvar('pscale')
+            r_list.append(r)
+            g_list.append(g)
+        print("len(r_list)", len(r_list))
+        print("len(g_list)", len(g_list))
+
+        # get results
+        self.rcalc = r_list[0].tolist()  # r0, r1, r2 are the same, so just use r0
+        # print self.rcalc
+        # sum up multi-phase PDFs
+        gsum = 0
+        for i in range(len(self.owner.strucs)):
+            gsum += g_list[i]
+        self.Gcalc = gsum.tolist()
+
         # structure needs to be read before dataset allocation
         for struc in self.owner.strucs:
             server.read_struct_string(struc.writeStr('pdffit'))

From f831795a6248bdd8bd4f18780b8698e1e376377b Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Wed, 6 Jan 2021 07:49:51 -0500
Subject: [PATCH 03/20] MNT: separate the cmi and pdffit calculation functions

---
 src/diffpy/pdfgui/control/calculation.py | 39 ++++++++++++++++++------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index ef78ce97..fd2e4c53 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -26,6 +26,7 @@
 from diffpy.pdfgui.utils import safeCPickleDumps, pickle_loads
 
 from diffpy.srreal.pdfcalculator import PDFCalculator, DebyePDFCalculator
+from diffpy.srreal.structureadapter import nometa
 
 
 class Calculation(PDFComponent):
@@ -138,8 +139,9 @@ def start(self):
             gui.postEvent(gui.PLOTNOW, self)
         return
 
-    def calculate(self):
-        """do the real calculation
+    def calculate_cmi(self):
+        """
+        Perform calculation in diffpy.srreal
         """
         # clean up old results
         self.rcalc = []
@@ -148,16 +150,17 @@ def calculate(self):
         # do the job
         if len(self.owner.strucs) == 0:
             raise ControlConfigError("No structure is given for calculation")
-
         # make sure parameters are initialized
         self.owner.updateParameters()
-        from diffpy.pdffit2 import PdfFit
-        server = PdfFit()
 
+        # initialize the calculator
         if self.pctype == 'PC': # use PDFCalculator
             pc = PDFCalculator()
         elif self.pctype == 'DPC': # use DebyePDFCalculator
             pc = DebyePDFCalculator()
+        # x-ray or neutron, PC default x-ray
+        if self.stye == "N":
+            pc.scatteringfactortable = "neutron"
 
         pc.qmax = self.qmax
         pc.qmin = self.qmin
@@ -178,9 +181,6 @@ def calculate(self):
             pc_temp = pc.copy()
             pc_temp.delta1 = struc.getvar('delta1')
             pc_temp.delta2 = struc.getvar('delta2')
-            if struc.getvar('pscale'):
-                pc_temp.addEnvelope('scale')
-                pc_temp.scale = struc.getvar('pscale')
             if struc.getvar('spdiameter'):
                 pc_temp.addEnvelope('sphericalshape')
                 pc_temp.spdiameter = struc.getvar('spdiameter')
@@ -194,21 +194,40 @@ def calculate(self):
             # pc_temp.setTypeMask("Ni","O",True)
 
             r, g = pc_temp(nometa(struc))
-            g = g * struc.getvar('pscale')
+            if struc.getvar('pscale'):
+                g = g * struc.getvar('pscale')
             r_list.append(r)
             g_list.append(g)
         print("len(r_list)", len(r_list))
         print("len(g_list)", len(g_list))
 
         # get results
+
         self.rcalc = r_list[0].tolist()  # r0, r1, r2 are the same, so just use r0
-        # print self.rcalc
         # sum up multi-phase PDFs
         gsum = 0
         for i in range(len(self.owner.strucs)):
             gsum += g_list[i]
         self.Gcalc = gsum.tolist()
 
+        return
+
+    def calculate(self):
+        """do the real calculation
+        """
+        # clean up old results
+        self.rcalc = []
+        self.Gcalc = []
+
+        # do the job
+        if len(self.owner.strucs) == 0:
+            raise ControlConfigError("No structure is given for calculation")
+
+        # make sure parameters are initialized
+        self.owner.updateParameters()
+        from diffpy.pdffit2 import PdfFit
+        server = PdfFit()
+
         # structure needs to be read before dataset allocation
         for struc in self.owner.strucs:
             server.read_struct_string(struc.writeStr('pdffit'))

From 92641f3a0e49736ec27cb44047436a737ae2eb3f Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 14 Jan 2021 08:13:11 -0500
Subject: [PATCH 04/20] MNT: add pctype and qmin configs in the calculation
 class.

---
 src/diffpy/pdfgui/control/calculation.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index fd2e4c53..38c5cbc8 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -295,6 +295,11 @@ def writeStr(self):
             lines.append('stype=X  x-ray scattering')
         elif self.stype == 'N':
             lines.append('stype=N  neutron scattering')
+        # pctype
+        if self.pctype == 'PC':
+            lines.append('pctype=PC  real space PDF calculator')
+        elif self.pctype == 'DPC':
+            lines.append('pctype=DPC  Debye PDF calculator')
         # dscale
         if self.dscale:
             lines.append('dscale=%g' % self.dscale)
@@ -304,6 +309,12 @@ def writeStr(self):
         else:
             qmax_line = 'qmax=%.2f' % self.qmax
         lines.append(qmax_line)
+        # qmin
+        if self.qmin == 0:
+            qmin_line = 'qmin=0'
+        else:
+            qmin_line = 'qmin=%.2f' % self.qmin
+        lines.append(qmin_line)
         # qdamp
         if isinstance(self.qdamp, float):
             lines.append('qdamp=%g' % self.qdamp)
@@ -335,7 +346,9 @@ def load(self, z, subpath):
         self.rcalc = config['rcalc']
         self.Gcalc = config['Gcalc']
         self.stype = config['stype']
+        self.pctype = config['pctype']
         self.qmax = config['qmax']
+        self.qmin = config['qmin']
         self.qdamp = config.get('qdamp', config.get('qsig'))
         self.qbroad = config.get('qbroad', config.get('qalp', 0.0))
         self.spdiameter = config.get('spdiameter')
@@ -356,7 +369,9 @@ def save(self, z, subpath):
             'rcalc' : self.rcalc,
             'Gcalc' : self.Gcalc,
             'stype' : self.stype,
+            'pctype' : self.pctype,
             'qmax' : self.qmax,
+            'qmin' : self.qmin,
             'qdamp' : self.qdamp,
             'qbroad' : self.qbroad,
             'dscale' : self.dscale,
@@ -377,7 +392,7 @@ def copy(self, other=None):
         # rcalc and Gcalc may be assigned, they get replaced by new lists
         # after every calculation
         assign_attributes = ( 'rmin', 'rstep', 'rmax', 'rlen',
-                'rcalc', 'Gcalc', 'stype', 'qmax', 'qdamp',
+                'rcalc', 'Gcalc', 'stype', 'pctype', 'qmax', 'qmin', 'qdamp',
                 'qbroad', 'dscale', )
         copy_attributes = ( )
         for a in assign_attributes:

From 00abb668c60850b72912727a3ad10c4f9e41f540 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 14 Jan 2021 08:44:52 -0500
Subject: [PATCH 05/20] ENH: add pair selection to calculator.

Use `pc.setTypeMask` method in `fitstructure.py`.
In `calculation.py`, apply it before calculate PDF.

TODO: the resulted baseline is different from PDFFIT.
---
 src/diffpy/pdfgui/control/calculation.py  |  9 ++++++---
 src/diffpy/pdfgui/control/fitstructure.py | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index 38c5cbc8..52eb9921 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -42,8 +42,11 @@ class Calculation(PDFComponent):
     rcalc  -- list of r values, this is set after calculation is finished
     Gcalc  -- list of calculated G values
     stype  -- scattering type, 'X' or 'N'
+    pctype -- PDF calculator type, 'PC' or 'DPC', to use RealSpacePC
+              or DebyePC.
     qmax   -- maximum value of Q in inverse Angstroms.  Termination ripples
               are ignored for qmax=0.
+    qmin   -- minimum value of Q in inverse Angstroms.  Default qmin=0.
     qdamp  -- specifies width of Gaussian damping factor in pdf_obs due
               to imperfect Q resolution
     qbroad -- quadratic peak broadening factor related to dataset
@@ -66,6 +69,7 @@ def __init__(self, name):
         self.rcalc = []
         self.Gcalc = []
         self.stype = 'X'
+        self.pctype = 'PC' # default use PDF real space calculator
         # user must specify qmax to get termination ripples
         self.qmax = 0.0
         self.qmin = 0.0
@@ -73,7 +77,6 @@ def __init__(self, name):
         self.qbroad = 0.0
         self.spdiameter = None
         self.dscale = 1.0
-        self.pctype = 'PC' # default use PDF real space calculator
         return
 
     def _getStrId(self):
@@ -159,7 +162,7 @@ def calculate_cmi(self):
         elif self.pctype == 'DPC': # use DebyePDFCalculator
             pc = DebyePDFCalculator()
         # x-ray or neutron, PC default x-ray
-        if self.stye == "N":
+        if self.stype == "N":
             pc.scatteringfactortable = "neutron"
 
         pc.qmax = self.qmax
@@ -190,7 +193,7 @@ def calculate_cmi(self):
 
 
             ##TODO: pair mask. the baseline is not correct with PDFFIT.
-            pc_temp = struc.applyCMIPairSelection(pc_temp)
+            struc.applyCMIPairSelection(pc_temp)
             # pc_temp.setTypeMask("Ni","O",True)
 
             r, g = pc_temp(nometa(struc))
diff --git a/src/diffpy/pdfgui/control/fitstructure.py b/src/diffpy/pdfgui/control/fitstructure.py
index 0dc52c8f..9311a648 100644
--- a/src/diffpy/pdfgui/control/fitstructure.py
+++ b/src/diffpy/pdfgui/control/fitstructure.py
@@ -621,6 +621,25 @@ def applyPairSelection(self, server, phaseidx):
             server.selectAtomIndex(phaseidx, 'j', idx, jflag)
         return
 
+    def applyCMIPairSelection(self, pc):
+        """Apply pair selection for CMI calculations of partial PDF.
+
+        pc   -- instance of CMI PDFCalculator engine
+        """
+        psf = self.getPairSelectionFlags()
+        # to begin masking, first remove ALL
+        pc.setTypeMask("all", "all", False)
+        #option1: mask by site index. has problem
+        # for i, iflag in enumerate(psf['firstflags']):
+        #     for j, jflag in enumerate(psf['secondflags']):
+        #         print("i, iflag", i, iflag)
+        #         print("j, jflag", j, jflag)
+        #         pc.setPairMask(i, j, (iflag and jflag))
+        #option2: mask by element type
+        firstelement = str(psf['fixed_pair_string'].split("-")[0])
+        secondelement = str(psf['fixed_pair_string'].split("-")[1])
+        pc.setTypeMask(firstelement, secondelement, True)
+        return
 
     def getSelectedIndices(self, s):
         '''Indices of the atoms that match the specified selection string.

From 7b0a8e708d5c7f21d2a3f5493940f6186f7def92 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 03:20:19 -0500
Subject: [PATCH 06/20] MNT: rename PDFFIT calculate function name.

---
 src/diffpy/pdfgui/control/calculation.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index 52eb9921..dc164752 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -142,7 +142,7 @@ def start(self):
             gui.postEvent(gui.PLOTNOW, self)
         return
 
-    def calculate_cmi(self):
+    def calculate(self):
         """
         Perform calculation in diffpy.srreal
         """
@@ -162,7 +162,7 @@ def calculate_cmi(self):
         elif self.pctype == 'DPC': # use DebyePDFCalculator
             pc = DebyePDFCalculator()
         # x-ray or neutron, PC default x-ray
-        if self.stype == "N":
+        if self.stype == 'N':
             pc.scatteringfactortable = "neutron"
 
         pc.qmax = self.qmax
@@ -215,8 +215,8 @@ def calculate_cmi(self):
 
         return
 
-    def calculate(self):
-        """do the real calculation
+    def calculate_pdffit(self):
+        """do the real calculation in PDFFIT
         """
         # clean up old results
         self.rcalc = []

From 95170af0fdb8296aeffb9b3dc8e051f51d3af36e Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 03:25:03 -0500
Subject: [PATCH 07/20] ENH: create cmi server in fitting.py.

Create `cmiserver` in `getServer` function.
Create `cmirecipe` in `configure` function.
Pass structure and data into `cmiserver` in `cmirecipe`.
---
 src/diffpy/pdfgui/control/fitting.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 7e42b1dc..bca76cea 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -123,6 +123,8 @@ def __init__(self, name):
 
         # the PDFfit2 server instance.
         self.server = None
+        # the CMI server instance.
+        self.cmiserver = None
 
         # public data members
         self.step = 0
@@ -157,6 +159,8 @@ def _release(self):
         """release resources"""
         if self.server: # server has been allocated, we need free the memory
             self.server.reset()
+        if self.cmiserver:
+            self.ciserver.reset()
 
     def _getStrId(self):
         """make a string identifier
@@ -320,6 +324,8 @@ def getServer(self):
         # create a new instance of calculation server
         from diffpy.pdffit2 import PdfFit
         self.server = PdfFit()
+        from diffpy.srfit.pdf import PDFContribution
+        self.cmiserver = PDFContribution("cmi")
         self.__changeStatus(fitStatus=Fitting.CONNECTED)
 
 
@@ -330,6 +336,17 @@ def configure(self):
 
         # make sure parameters are initialized
         self.updateParameters()
+
+        from diffpy.srreal.structureadapter import nometa
+        from diffpy.srfit.fitbase import FitRecipe, FitResults
+        from scipy.optimize.minpack import leastsq
+        self.cmiserver.addStructure("cmi",self.strucs[0])
+        self.cmiserver.loadData(self.datasets[0].writeResampledObsStr())
+        cmirecipe = FitRecipe()
+        cmirecipe.addContribution(self.cmiserver)
+
+
+
         self.server.reset()
         for struc in self.strucs:
             struc.clearRefined()

From 800a6e4a2e8c7aeee5e439341e79021898479437 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 04:00:31 -0500
Subject: [PATCH 08/20] ENH: add DPC/PC and Qmin in `pdfdataset`.

Add `pctype` and `qmin` in pdfdataset.py.
---
 src/diffpy/pdfgui/control/pdfdataset.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/diffpy/pdfgui/control/pdfdataset.py b/src/diffpy/pdfgui/control/pdfdataset.py
index 48fb53e0..e71ddfed 100644
--- a/src/diffpy/pdfgui/control/pdfdataset.py
+++ b/src/diffpy/pdfgui/control/pdfdataset.py
@@ -35,8 +35,11 @@ class PDFDataSet(PDFComponent):
         drobs      -- list of standard deviations of robs
         dGobs      -- list of standard deviations of Gobs
         stype      -- scattering type, 'X' or 'N'
+        pctype     -- PDF calculator type, 'PC' or 'DPC', to use RealSpacePC
+                      or DebyePC.
         qmax       -- maximum value of Q in inverse Angstroms.  Termination
                       ripples are neglected for qmax=0.
+        qmin       -- minimum value of Q in inverse Angstroms.  Default qmin=0.
         qdamp      -- specifies width of Gaussian damping factor in pdf_obs due
                       to imperfect Q resolution
         qbroad     -- quadratic peak broadening factor related to dataset
@@ -56,7 +59,7 @@ class PDFDataSet(PDFComponent):
         refinableVars   -- set (dict) of refinable variable names.
     """
 
-    persistentItems = [ 'robs', 'Gobs', 'drobs', 'dGobs', 'stype', 'qmax',
+    persistentItems = [ 'robs', 'Gobs', 'drobs', 'dGobs', 'stype', 'pctype', 'qmax', 'qmin',
                      'qdamp', 'qbroad', 'dscale', 'rmin', 'rmax', 'metadata' ]
     refinableVars = dict.fromkeys(('qdamp', 'qbroad', 'dscale'))
 
@@ -77,6 +80,8 @@ def clear(self):
         self.dGobs = []
         self.stype = 'X'
         # user must specify qmax to get termination ripples
+        self.pctype = 'PC'
+        self.qmin = 0.0
         self.qmax = 0.0
         self.qdamp = 0.001
         self.qbroad = 0.0
@@ -216,7 +221,7 @@ def readStr(self, datastring):
         if res:
             self.metadata['doping'] = float(res.groups()[0])
 
-        # parsing gerneral metadata
+        # parsing general metadata
         if metadata:
             regexp = r"\b(\w+)\ *=\ *(%(f)s)\b" % rx
             while True:

From 707b33bb28fb75c7f145afcf11dbd4f03bd4df7b Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 04:19:00 -0500
Subject: [PATCH 09/20] ENH: add DPC/PC in fitting.

Add `DebyePDFGenerator` and `PDFGenerator` in fitting.
`self.datasets[0].pctype` is GUI choice of DPC/PC.
Use contribution instead of PDFcontribution.
---
 src/diffpy/pdfgui/control/fitting.py | 51 ++++++++++++++++++++++++----
 1 file changed, 45 insertions(+), 6 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index bca76cea..8b2da7f9 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -123,8 +123,13 @@ def __init__(self, name):
 
         # the PDFfit2 server instance.
         self.server = None
+
         # the CMI server instance.
-        self.cmiserver = None
+        self.cmipdfgen = None
+        self.cmiprofile = None
+        self.cmicontribution = None
+        self.cmirecipe = None
+        self.cmiresults = None
 
         # public data members
         self.step = 0
@@ -324,8 +329,11 @@ def getServer(self):
         # create a new instance of calculation server
         from diffpy.pdffit2 import PdfFit
         self.server = PdfFit()
+
         from diffpy.srfit.pdf import PDFContribution
-        self.cmiserver = PDFContribution("cmi")
+        from diffpy.srfit.fitbase import Profile
+        self.cmiprofile = Profile()
+
         self.__changeStatus(fitStatus=Fitting.CONNECTED)
 
 
@@ -338,19 +346,50 @@ def configure(self):
         self.updateParameters()
 
         from diffpy.srreal.structureadapter import nometa
+        from diffpy.srfit.fitbase import FitContribution, FitRecipe, FitResults
+        from diffpy.srfit.pdf import PDFParser
+        from diffpy.srfit.pdf import PDFGenerator, DebyePDFGenerator
         from diffpy.srfit.fitbase import FitRecipe, FitResults
         from scipy.optimize.minpack import leastsq
-        self.cmiserver.addStructure("cmi",self.strucs[0])
-        self.cmiserver.loadData(self.datasets[0].writeResampledObsStr())
+
+        if self.datasets[0].pctype == 'PC':
+            self.cmipdfgen = PDFGenerator("cmipdfgen")
+        elif self.datasets[0].pctype == 'DPC':
+            self.cmipdfgen = DebyePDFGenerator("cmipdfgen")
+
+        self.cmipdfgen.setStructure(self.strucs[0])
+        parser = PDFParser()
+        parser.parseString(self.datasets[0].writeResampledObsStr())
+        self.cmiprofile.loadParsedData(parser)
+
+        self.cmicontribution = FitContribution("cmicontribution")
+        self.cmicontribution.addProfileGenerator(self.cmipdfgen)
+        self.cmicontribution.setProfile(self.cmiprofile, xname ="r")
+        self.cmicontribution.setEquation("scale * cmipdfgen")
+
+        # add qmax, qdamp, qbroad into cmipdfgen
+        self.cmipdfgen.setQmax(self.datasets[0].qmax)
+        self.cmipdfgen.qdamp.value = self.datasets[0].qdamp
+        self.cmipdfgen.qbroad.value = self.datasets[0].qbroad
+
         cmirecipe = FitRecipe()
-        cmirecipe.addContribution(self.cmiserver)
+        cmirecipe.addContribution(self.cmicontribution)
+        self.cmirecipe.addVar(self.cmicontribution.scale, 1.0)
+        # turn on printout fithook in each refinement step
+        self.cmirecipe.fithooks[0].verbose = 3
+
+        leastsq(self.cmirecipe.residual, self.cmirecipe.values)
+        self.cmiresults = str(FitResults(self.cmirecipe))
+
+
+
 
 
 
         self.server.reset()
         for struc in self.strucs:
             struc.clearRefined()
-            self.server.read_struct_string(struc.initial.writeStr("pdffit") )
+            self.server.read_struct_string(struc.initial.writeStr("pdffit"))
             for key, var in struc.constraints.items():
                 self.server.constrain(key, var.formula)
 

From e19eb22954abba52390e7c0bc64731fb28dd2558 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 05:05:28 -0500
Subject: [PATCH 10/20] ENH: constrain pscale and lattice.

Pass `pscale` and `lat(n)` parameters and their initial values to cmirecipe in `fitting.py`.
---
 src/diffpy/pdfgui/control/fitting.py | 63 +++++++++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 8b2da7f9..8f04ba04 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -345,6 +345,7 @@ def configure(self):
         # make sure parameters are initialized
         self.updateParameters()
 
+        self.applyParameters()
         from diffpy.srreal.structureadapter import nometa
         from diffpy.srfit.fitbase import FitContribution, FitRecipe, FitResults
         from diffpy.srfit.pdf import PDFParser
@@ -372,9 +373,63 @@ def configure(self):
         self.cmipdfgen.qdamp.value = self.datasets[0].qdamp
         self.cmipdfgen.qbroad.value = self.datasets[0].qbroad
 
+        lat = self.cmipdfgen.phase.getLattice()
+        atoms = self.cmipdfgen.phase.getScatterers()
+
         cmirecipe = FitRecipe()
         cmirecipe.addContribution(self.cmicontribution)
-        self.cmirecipe.addVar(self.cmicontribution.scale, 1.0)
+        # self.cmirecipe.addVar(self.cmicontribution.scale, 1.0)
+
+        for index, par in self.parameters.items():
+            # clean any refined value
+            par.refined = None
+            # self.server.setpar(index, par.initialValue()) # info[0] = init value
+            var_name = "var" + str(index)
+            print("var_name")
+            print(var_name)
+            print("par.initialValue()")
+            print(par.initialValue())
+            self.cmirecipe.newVar(var_name, par.initialValue())
+            # fix if fixed.  Note: all parameters are free after self.server.reset().
+            # if par.fixed:
+            #     self.server.fixpar(index)
+
+        for struc in self.strucs:
+            struc.clearRefined()
+            for key, var in struc.constraints.items():
+                # self.server.constrain(key, var.formula)
+                var_name = self.transVar(var.formula)
+                print("var_name here")
+                print(var_name)
+                print("key", key, type(key))
+                if key == 'pscale':
+                    self.cmirecipe.constrain(self.cmicontribution.scale, var_name)
+                if key == 'lat(1)':
+                    self.cmirecipe.constrain(lat.a, var_name)
+                if key == 'lat(2)':
+                    self.cmirecipe.constrain(lat.b, var_name)
+                if key == 'lat(3)':
+                    self.cmirecipe.constrain(lat.c, var_name)
+                if key == 'lat(4)':
+                    self.cmirecipe.constrain(lat.alpha, var_name)
+                if key == 'lat(5)':
+                    self.cmirecipe.constrain(lat.beta, var_name)
+                if key == 'lat(6)':
+                    self.cmirecipe.constrain(lat.gamma, var_name)
+
+                # ADP
+                ## TODO key == 'u11(i)', constrain the ith atom's ADP U11.
+                # How to determine key == 'u11(i)' and parse the number 'i' in ().
+                # if key == 'u11(i)':
+                    # self.cmirecipe.constrain(atoms[i-1].U11, var_name)
+
+                # delta term
+                if key == 'delta2':
+                    self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
+                if key == 'delta1':
+                    self.cmirecipe.constrain(self.cmipdfgen.delta1, var_name)
+
+
         # turn on printout fithook in each refinement step
         self.cmirecipe.fithooks[0].verbose = 3
 
@@ -865,4 +920,10 @@ def _getData(self, id, name, step=-1):
         else:
             return self.snapshots[step][index]
 
+    # Long new helper function
+    def transVar(self, str):
+        # input "@11"
+        # output "var11"
+        return str.replace("@", "var")
+
 # End of file

From 31461fb29f481b45d57a996ca72e269a34fe2a39 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 06:27:39 -0500
Subject: [PATCH 11/20] ENH: update robust parser for `key` in fitting.

Copy `__getRef` function from PDFfit2.pdffit.py to `fitting.py`, for getting parameters from `key`.
For example, `lat(n)`(key) can now be parsed into `lat`(key_ref) and `n`(key_arg) separately.
---
 src/diffpy/pdfgui/control/fitting.py | 82 +++++++++++++++++++++-------
 1 file changed, 62 insertions(+), 20 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 8f04ba04..8b95acd1 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -164,8 +164,6 @@ def _release(self):
         """release resources"""
         if self.server: # server has been allocated, we need free the memory
             self.server.reset()
-        if self.cmiserver:
-            self.ciserver.reset()
 
     def _getStrId(self):
         """make a string identifier
@@ -376,8 +374,8 @@ def configure(self):
         lat = self.cmipdfgen.phase.getLattice()
         atoms = self.cmipdfgen.phase.getScatterers()
 
-        cmirecipe = FitRecipe()
-        cmirecipe.addContribution(self.cmicontribution)
+        self.cmirecipe = FitRecipe()
+        self.cmirecipe.addContribution(self.cmicontribution)
         # self.cmirecipe.addVar(self.cmicontribution.scale, 1.0)
 
         for index, par in self.parameters.items():
@@ -398,24 +396,28 @@ def configure(self):
             struc.clearRefined()
             for key, var in struc.constraints.items():
                 # self.server.constrain(key, var.formula)
+                key_ref, key_arg = self.__getRef(key)
                 var_name = self.transVar(var.formula)
                 print("var_name here")
                 print(var_name)
                 print("key", key, type(key))
-                if key == 'pscale':
+                print("key_ref", key_ref)
+                print("key_arg", key_arg)
+                if key_ref == 'pscale':
                     self.cmirecipe.constrain(self.cmicontribution.scale, var_name)
-                if key == 'lat(1)':
-                    self.cmirecipe.constrain(lat.a, var_name)
-                if key == 'lat(2)':
-                    self.cmirecipe.constrain(lat.b, var_name)
-                if key == 'lat(3)':
-                    self.cmirecipe.constrain(lat.c, var_name)
-                if key == 'lat(4)':
-                    self.cmirecipe.constrain(lat.alpha, var_name)
-                if key == 'lat(5)':
-                    self.cmirecipe.constrain(lat.beta, var_name)
-                if key == 'lat(6)':
-                    self.cmirecipe.constrain(lat.gamma, var_name)
+                if key_ref == 'lat':
+                    if key_arg == '1':
+                        self.cmirecipe.constrain(lat.a, var_name)
+                    if key_arg == '2':
+                        self.cmirecipe.constrain(lat.b, var_name)
+                    if key_arg == '3':
+                        self.cmirecipe.constrain(lat.c, var_name)
+                    if key_arg == '4':
+                        self.cmirecipe.constrain(lat.alpha, var_name)
+                    if key_arg == '5':
+                        self.cmirecipe.constrain(lat.beta, var_name)
+                    if key_arg == '6':
+                        self.cmirecipe.constrain(lat.gamma, var_name)
 
                 # ADP
                 ## TODO key == 'u11(i)', constrain the ith atom's ADP U11.
@@ -424,10 +426,11 @@ def configure(self):
                     # self.cmirecipe.constrain(atoms[i-1].U11, var_name)
 
                 # delta term
-                if key == 'delta2':
-                    self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
-                if key == 'delta1':
+                if key_ref == 'delta1':
                     self.cmirecipe.constrain(self.cmipdfgen.delta1, var_name)
+                if key_ref == 'delta2':
+                    self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
+
 
 
         # turn on printout fithook in each refinement step
@@ -926,4 +929,43 @@ def transVar(self, str):
         # output "var11"
         return str.replace("@", "var")
 
+    def __getRef(self, var_string):
+        # copy from __getRef in pdffit.py from PDFfit2 package.
+        # input pscale, output method_string = "pscale", arg_int = None
+        # input lat(1), output method_string = "lat", arg_int = 1
+        # input u11(2), output method_string = "u11", arg_int = 2
+        """Return the actual reference to the variable in the var_string.
+
+        This function must be called before trying to actually reference an
+        internal variable. See the constrain method for an example.
+
+        Raises:
+            pdffit2.unassignedError if variable is not yet assigned
+            ValueError if variable index does not exist (e.g. lat(7))
+        """
+        var_string = _convertCallable(var_string)
+        arg_int = None
+        try:
+            method_string, arg_string = var_string.split("(")
+            method_string = method_string.strip()
+            arg_int = int(arg_string.strip(")").strip())
+        except ValueError: #There is no arg_string
+            method_string = var_string.strip()
+        return method_string, arg_int
+
+
+# Long helper routines
+def _convertCallable(var):
+     """Convert an object to the result of its call when callable.
+
+     var -- string or callable object that returns string
+
+     Return var or var().
+     """
+     if callable(var):
+         rv = var()
+     else:
+         rv = var
+     return rv
+
 # End of file

From 24c5f5c8bc6535e97974c3916ed44feed59ec3a9 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 06:45:41 -0500
Subject: [PATCH 12/20] ENH: constrain ADP, xyz, occupancy.

Pass `Uij`, `xyz`,`occu` parameters and their initial values to cmirecipe in `fitting.py`.
---
 src/diffpy/pdfgui/control/fitting.py | 32 ++++++++++++++++++++++------
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 8b95acd1..5134b608 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -419,18 +419,38 @@ def configure(self):
                     if key_arg == '6':
                         self.cmirecipe.constrain(lat.gamma, var_name)
 
-                # ADP
-                ## TODO key == 'u11(i)', constrain the ith atom's ADP U11.
-                # How to determine key == 'u11(i)' and parse the number 'i' in ().
-                # if key == 'u11(i)':
-                    # self.cmirecipe.constrain(atoms[i-1].U11, var_name)
-
                 # delta term
                 if key_ref == 'delta1':
                     self.cmirecipe.constrain(self.cmipdfgen.delta1, var_name)
                 if key_ref == 'delta2':
                     self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
 
+                # ADP
+                ## TODO key_ascii == 'u11(i)', constrain the ith atom's ADP U11.
+                if key_ref == 'u11':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U11, var_name)
+                if key_ref == 'u22':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U22, var_name)
+                if key_ref == 'u33':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U33, var_name)
+                if key_ref == 'u12':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U12, var_name)
+                if key_ref == 'u13':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U13, var_name)
+                if key_ref == 'u23':
+                    self.cmirecipe.constrain(atoms[key_arg-1].U23, var_name)
+
+                # atom positions
+                if key_ref == 'x':
+                    self.cmirecipe.constrain(atoms[key_arg-1].x, var_name)
+                if key_ref == 'y':
+                    self.cmirecipe.constrain(atoms[key_arg-1].y, var_name)
+                if key_ref == 'z':
+                    self.cmirecipe.constrain(atoms[key_arg-1].z, var_name)
+
+                # occupancy
+                if key_ref == 'occ':
+                    self.cmirecipe.constrain(atoms[key_arg-1].occupancy, var_name)
 
 
         # turn on printout fithook in each refinement step

From 0f0ecfa3113397632feb646adfa6ef477149d4bd Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 06:58:40 -0500
Subject: [PATCH 13/20] MNT: refactor cmi constrain codes in a separate
 function.

Put parameter constrain codes into `cmiConstrain` function in `fitting.py`.
---
 src/diffpy/pdfgui/control/fitting.py | 125 +++++++++++++++------------
 1 file changed, 68 insertions(+), 57 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 5134b608..9672ee0a 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -371,8 +371,6 @@ def configure(self):
         self.cmipdfgen.qdamp.value = self.datasets[0].qdamp
         self.cmipdfgen.qbroad.value = self.datasets[0].qbroad
 
-        lat = self.cmipdfgen.phase.getLattice()
-        atoms = self.cmipdfgen.phase.getScatterers()
 
         self.cmirecipe = FitRecipe()
         self.cmirecipe.addContribution(self.cmicontribution)
@@ -396,61 +394,8 @@ def configure(self):
             struc.clearRefined()
             for key, var in struc.constraints.items():
                 # self.server.constrain(key, var.formula)
-                key_ref, key_arg = self.__getRef(key)
-                var_name = self.transVar(var.formula)
-                print("var_name here")
-                print(var_name)
-                print("key", key, type(key))
-                print("key_ref", key_ref)
-                print("key_arg", key_arg)
-                if key_ref == 'pscale':
-                    self.cmirecipe.constrain(self.cmicontribution.scale, var_name)
-                if key_ref == 'lat':
-                    if key_arg == '1':
-                        self.cmirecipe.constrain(lat.a, var_name)
-                    if key_arg == '2':
-                        self.cmirecipe.constrain(lat.b, var_name)
-                    if key_arg == '3':
-                        self.cmirecipe.constrain(lat.c, var_name)
-                    if key_arg == '4':
-                        self.cmirecipe.constrain(lat.alpha, var_name)
-                    if key_arg == '5':
-                        self.cmirecipe.constrain(lat.beta, var_name)
-                    if key_arg == '6':
-                        self.cmirecipe.constrain(lat.gamma, var_name)
-
-                # delta term
-                if key_ref == 'delta1':
-                    self.cmirecipe.constrain(self.cmipdfgen.delta1, var_name)
-                if key_ref == 'delta2':
-                    self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
-
-                # ADP
-                ## TODO key_ascii == 'u11(i)', constrain the ith atom's ADP U11.
-                if key_ref == 'u11':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U11, var_name)
-                if key_ref == 'u22':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U22, var_name)
-                if key_ref == 'u33':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U33, var_name)
-                if key_ref == 'u12':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U12, var_name)
-                if key_ref == 'u13':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U13, var_name)
-                if key_ref == 'u23':
-                    self.cmirecipe.constrain(atoms[key_arg-1].U23, var_name)
-
-                # atom positions
-                if key_ref == 'x':
-                    self.cmirecipe.constrain(atoms[key_arg-1].x, var_name)
-                if key_ref == 'y':
-                    self.cmirecipe.constrain(atoms[key_arg-1].y, var_name)
-                if key_ref == 'z':
-                    self.cmirecipe.constrain(atoms[key_arg-1].z, var_name)
-
-                # occupancy
-                if key_ref == 'occ':
-                    self.cmirecipe.constrain(atoms[key_arg-1].occupancy, var_name)
+
+                self.cmiConstrain(key, var)
 
 
         # turn on printout fithook in each refinement step
@@ -944,6 +889,72 @@ def _getData(self, id, name, step=-1):
             return self.snapshots[step][index]
 
     # Long new helper function
+    def cmiConstrain(self, key, var):
+        """Constrain structure parameters into cmi receipe.
+        key -- names of parameters, like pscale, lat(n).
+        var -- var.formula represents names of constrains, like @1, @2 + 1.
+        """
+        key_ref, key_arg = self.__getRef(key)
+        var_name = self.transVar(var.formula)
+
+        lat = self.cmipdfgen.phase.getLattice()
+        atoms = self.cmipdfgen.phase.getScatterers()
+
+        print("var_name here")
+        print(var_name)
+        print("key", key, type(key))
+        print("key_ref", key_ref)
+        print("key_arg", key_arg)
+        if key_ref == 'pscale':
+            self.cmirecipe.constrain(self.cmicontribution.scale, var_name)
+        if key_ref == 'lat':
+            if key_arg == '1':
+                self.cmirecipe.constrain(lat.a, var_name)
+            if key_arg == '2':
+                self.cmirecipe.constrain(lat.b, var_name)
+            if key_arg == '3':
+                self.cmirecipe.constrain(lat.c, var_name)
+            if key_arg == '4':
+                self.cmirecipe.constrain(lat.alpha, var_name)
+            if key_arg == '5':
+                self.cmirecipe.constrain(lat.beta, var_name)
+            if key_arg == '6':
+                self.cmirecipe.constrain(lat.gamma, var_name)
+
+        # delta term
+        if key_ref == 'delta1':
+            self.cmirecipe.constrain(self.cmipdfgen.delta1, var_name)
+        if key_ref == 'delta2':
+            self.cmirecipe.constrain(self.cmipdfgen.delta2, var_name)
+
+        # ADP
+        ## TODO key_ascii == 'u11(i)', constrain the ith atom's ADP U11.
+        if key_ref == 'u11':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U11, var_name)
+        if key_ref == 'u22':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U22, var_name)
+        if key_ref == 'u33':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U33, var_name)
+        if key_ref == 'u12':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U12, var_name)
+        if key_ref == 'u13':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U13, var_name)
+        if key_ref == 'u23':
+            self.cmirecipe.constrain(atoms[key_arg - 1].U23, var_name)
+
+        # atom positions
+        if key_ref == 'x':
+            self.cmirecipe.constrain(atoms[key_arg - 1].x, var_name)
+        if key_ref == 'y':
+            self.cmirecipe.constrain(atoms[key_arg - 1].y, var_name)
+        if key_ref == 'z':
+            self.cmirecipe.constrain(atoms[key_arg - 1].z, var_name)
+
+        # occupancy
+        if key_ref == 'occ':
+            self.cmirecipe.constrain(atoms[key_arg - 1].occupancy, var_name)
+        return
+
     def transVar(self, str):
         # input "@11"
         # output "var11"

From 7a6f39a1a70ed71b5880a8448765310d000c976a Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:00:15 -0500
Subject: [PATCH 14/20] ENH: Pass fit data range in fitting.

Fitrmin, fitrmax, and fitrstep are in `FitDataSet` object.
---
 src/diffpy/pdfgui/control/fitting.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 9672ee0a..8149faf1 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -360,6 +360,9 @@ def configure(self):
         parser = PDFParser()
         parser.parseString(self.datasets[0].writeResampledObsStr())
         self.cmiprofile.loadParsedData(parser)
+        self.cmiprofile.setCalculationRange(xmin = self.datasets[0].fitrmin,
+                                            xmax = self.datasets[0].fitrmax,
+                                            dx = self.datasets[0].fitrstep)
 
         self.cmicontribution = FitContribution("cmicontribution")
         self.cmicontribution.addProfileGenerator(self.cmipdfgen)

From c961a2ef0ee6a645d18760b5e7adbb356a62d580 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:13:39 -0500
Subject: [PATCH 15/20] ENH: constrain qdamp and qbroad.

Pass `qdamp` and `qbroad` parameters and their initial values to cmirecipe in `fitting.py`.
---
 src/diffpy/pdfgui/control/fitting.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 8149faf1..bed65f5d 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -393,12 +393,25 @@ def configure(self):
             # if par.fixed:
             #     self.server.fixpar(index)
 
+        # phase constrains
         for struc in self.strucs:
             struc.clearRefined()
             for key, var in struc.constraints.items():
-                # self.server.constrain(key, var.formula)
+                self.cmiConstrain(key, var)
 
+        # data constrains
+        for dataset in self.datasets:
+            dataset.clearRefined()
+            for key, var in dataset.constraints.items():
                 self.cmiConstrain(key, var)
+            # Removed call to pdfrange call, because data were already
+            # resampled to at fit range.
+            #
+            # Pair selection applies only to the current dataset,
+            # therefore it has to be done here.
+            nstrucs = len(self.strucs)
+            for phaseidx, struc in zip(range(1, nstrucs + 1), self.strucs):
+                struc.applyPairSelection(self.server, phaseidx)
 
 
         # turn on printout fithook in each refinement step
@@ -956,6 +969,14 @@ def cmiConstrain(self, key, var):
         # occupancy
         if key_ref == 'occ':
             self.cmirecipe.constrain(atoms[key_arg - 1].occupancy, var_name)
+
+        # data parameters
+        if key_ascii_ref == 'qdamp':
+            self.cmirecipe.constrain(self.cmipdfgen.qdamp, var_name)
+        if key_ascii_ref == 'qbroad':
+            self.cmirecipe.constrain(self.cmipdfgen.qbroad, var_name)
+        # TODO how to deal with `dscale`. cmipdfgen don't have `dscale` parameter.
+
         return
 
     def transVar(self, str):

From 90feeeab4e500e40cb908f403b6577cfc30e4cd7 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:20:18 -0500
Subject: [PATCH 16/20] ENH: show CMI results.

Hide PDFFIT result and replace with CMI result in the `outputpanel`.
---
 src/diffpy/pdfgui/control/calculation.py   |  1 +
 src/diffpy/pdfgui/control/fitting.py       | 13 +++++++++++--
 src/diffpy/pdfgui/control/pdfguicontrol.py | 14 ++++++++++++++
 src/diffpy/pdfgui/gui/mainframe.py         |  8 +++++++-
 4 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/src/diffpy/pdfgui/control/calculation.py b/src/diffpy/pdfgui/control/calculation.py
index dc164752..e81fc8ce 100644
--- a/src/diffpy/pdfgui/control/calculation.py
+++ b/src/diffpy/pdfgui/control/calculation.py
@@ -178,6 +178,7 @@ def calculate(self):
         # and set any calculator attributes as needed as above
         r_list = []
         g_list = []
+        self.owner.applyParameters()
         for struc in self.owner.strucs:
             # pc is for one calculation. the common setting
             # pc_temp is for each phase, specific setting
diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index bed65f5d..21ee9d64 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -418,8 +418,9 @@ def configure(self):
         self.cmirecipe.fithooks[0].verbose = 3
 
         leastsq(self.cmirecipe.residual, self.cmirecipe.values)
-        self.cmiresults = str(FitResults(self.cmirecipe))
-
+        self.cmiresults = "\n=============================== CMI RESULTS ==================================\n"
+        self.cmiresults += str(FitResults(self.cmirecipe))
+        self.cmiresults += "============================ END OF CMI RESULTS ==============================\n\n"
 
 
 
@@ -471,6 +472,14 @@ def resetStatus(self):
         """reset status back to initialized"""
         self.snapshots = []
         self.step = 0
+        # long
+        # initialize cmi
+        self.cmipdfgen = None
+        self.cmiprofile = None
+        self.cmicontribution = None
+        self.cmirecipe = None
+        self.cmiresults = None
+        # end long
         if self.fitStatus == Fitting.INITIALIZED:
             return  # already reset
 
diff --git a/src/diffpy/pdfgui/control/pdfguicontrol.py b/src/diffpy/pdfgui/control/pdfguicontrol.py
index 5f838a5f..012724da 100644
--- a/src/diffpy/pdfgui/control/pdfguicontrol.py
+++ b/src/diffpy/pdfgui/control/pdfguicontrol.py
@@ -529,6 +529,20 @@ def getEngineOutput(self):
         redirect_stdout(six.StringIO())
         return txt
 
+    #long
+    def getCMIOutput(self):
+        """Get the output from the CMI engine."""
+        txt = None
+        if self.currentFitting:
+            if self.currentFitting.cmiresults:
+                txt = self.currentFitting.cmiresults
+        return txt
+
+    def resetCMIOutput(self):
+        self.currentFitting.cmiresults = None
+        return
+    #end Long
+
 _pdfguicontrol = None
 def pdfguicontrol(*args, **kwargs):
     """This function will return the single instance of class PDFGuiControl"""
diff --git a/src/diffpy/pdfgui/gui/mainframe.py b/src/diffpy/pdfgui/gui/mainframe.py
index 24a0b578..afc074da 100644
--- a/src/diffpy/pdfgui/gui/mainframe.py
+++ b/src/diffpy/pdfgui/gui/mainframe.py
@@ -2497,7 +2497,13 @@ def updateFittingStatus(self, job):
 
     def updateOutput(self):
         """Update text in outputPanel with text in stdout."""
-        self.outputPanel.updateText(self.control.getEngineOutput())
+        # self.outputPanel.updateText(self.control.getEngineOutput())
+        #long
+        # TODO: append CMI result after pdffit result
+        if self.control.getCMIOutput():
+            self.outputPanel.updateText(self.control.getCMIOutput())
+            self.control.resetCMIOutput() #only output cmi results once
+        #end long
         return
 
 # end of class MainPanel

From 492f6f349b2141299cfd37d91e361ab20c8e9aa7 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:25:30 -0500
Subject: [PATCH 17/20] STY: change program name into `PDFgui 2.0`.

---
 src/diffpy/pdfgui/gui/pdfguiglobals.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/diffpy/pdfgui/gui/pdfguiglobals.py b/src/diffpy/pdfgui/gui/pdfguiglobals.py
index 406ed5ef..61948a22 100644
--- a/src/diffpy/pdfgui/gui/pdfguiglobals.py
+++ b/src/diffpy/pdfgui/gui/pdfguiglobals.py
@@ -13,13 +13,13 @@
 #
 ##############################################################################
 
-"""This module contains gloabal parameters needed by PDFgui."""
+"""This module contains global parameters needed by PDFgui."""
 
 import os.path
 from pkg_resources import Requirement, resource_filename
 
 # Name of the program
-name = "PDFgui"
+name = "PDFgui 2.0"
 # Maximum number of files to be remembered
 MAXMRU = 5
 # The location of the configuration file

From 0af780c3937b545aeab3305541930e9401e81fa0 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:29:02 -0500
Subject: [PATCH 18/20] STY: rename `outputpanel` caption.

After changing `mainframe.py`, one needs to hide the panel and reopen it in `View` Manu to update the changes.
---
 src/diffpy/pdfgui/gui/mainframe.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/diffpy/pdfgui/gui/mainframe.py b/src/diffpy/pdfgui/gui/mainframe.py
index afc074da..7f338c44 100644
--- a/src/diffpy/pdfgui/gui/mainframe.py
+++ b/src/diffpy/pdfgui/gui/mainframe.py
@@ -352,7 +352,7 @@ def __customProperties(self):
         # Position other panels. Note that currently MinimizeButton does not do
         # anything. It is to be implemented in future versions of wx.aui
         self.auiManager.AddPane(self.outputPanel, wx.aui.AuiPaneInfo().
-                          Name("outputPanel").Caption("PDFfit2 Output").
+                          Name("outputPanel").Caption("DiffPy-CMI Output").
                           Bottom().
                           TopDockable().
                           BottomDockable().

From 6a660e80d2e13219c53f289a51bebebd28da574d Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Thu, 4 Mar 2021 07:50:15 -0500
Subject: [PATCH 19/20] STY: move import packages to the top of the file.

---
 src/diffpy/pdfgui/control/fitting.py | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 21ee9d64..36e01a8d 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -24,6 +24,15 @@
 from diffpy.pdfgui.control.controlerrors import ControlValueError
 from diffpy.pdfgui.utils import safeCPickleDumps, pickle_loads
 
+from diffpy.srfit.pdf import PDFContribution
+from diffpy.srfit.fitbase import Profile
+from diffpy.srreal.structureadapter import nometa
+from diffpy.srfit.fitbase import FitContribution, FitRecipe, FitResults
+from diffpy.srfit.pdf import PDFParser
+from diffpy.srfit.pdf import PDFGenerator, DebyePDFGenerator
+from diffpy.srfit.fitbase import FitRecipe, FitResults
+from scipy.optimize.minpack import leastsq
+
 # helper routines to deal with PDFfit2 exceptions
 
 def getEngineExceptions():
@@ -328,8 +337,6 @@ def getServer(self):
         from diffpy.pdffit2 import PdfFit
         self.server = PdfFit()
 
-        from diffpy.srfit.pdf import PDFContribution
-        from diffpy.srfit.fitbase import Profile
         self.cmiprofile = Profile()
 
         self.__changeStatus(fitStatus=Fitting.CONNECTED)
@@ -343,13 +350,8 @@ def configure(self):
         # make sure parameters are initialized
         self.updateParameters()
 
+        #long CMI part
         self.applyParameters()
-        from diffpy.srreal.structureadapter import nometa
-        from diffpy.srfit.fitbase import FitContribution, FitRecipe, FitResults
-        from diffpy.srfit.pdf import PDFParser
-        from diffpy.srfit.pdf import PDFGenerator, DebyePDFGenerator
-        from diffpy.srfit.fitbase import FitRecipe, FitResults
-        from scipy.optimize.minpack import leastsq
 
         if self.datasets[0].pctype == 'PC':
             self.cmipdfgen = PDFGenerator("cmipdfgen")
@@ -374,7 +376,6 @@ def configure(self):
         self.cmipdfgen.qdamp.value = self.datasets[0].qdamp
         self.cmipdfgen.qbroad.value = self.datasets[0].qbroad
 
-
         self.cmirecipe = FitRecipe()
         self.cmirecipe.addContribution(self.cmicontribution)
         # self.cmirecipe.addVar(self.cmicontribution.scale, 1.0)
@@ -422,9 +423,7 @@ def configure(self):
         self.cmiresults += str(FitResults(self.cmirecipe))
         self.cmiresults += "============================ END OF CMI RESULTS ==============================\n\n"
 
-
-
-
+        #Long end CMI part
 
         self.server.reset()
         for struc in self.strucs:
@@ -925,11 +924,6 @@ def cmiConstrain(self, key, var):
         lat = self.cmipdfgen.phase.getLattice()
         atoms = self.cmipdfgen.phase.getScatterers()
 
-        print("var_name here")
-        print(var_name)
-        print("key", key, type(key))
-        print("key_ref", key_ref)
-        print("key_arg", key_arg)
         if key_ref == 'pscale':
             self.cmirecipe.constrain(self.cmicontribution.scale, var_name)
         if key_ref == 'lat':

From 928114b0065dfaa7d60c77d6379834b846aef8a3 Mon Sep 17 00:00:00 2001
From: Long Yang <ly2364@columbia.edu>
Date: Fri, 20 May 2022 10:51:10 +0800
Subject: [PATCH 20/20] MNT: fix py2 to py3 conversion errors for cmi engine.

Qdamp and Qbroad work in calculation, but do not work in fitting yet.
---
 src/diffpy/pdfgui/control/fitting.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/diffpy/pdfgui/control/fitting.py b/src/diffpy/pdfgui/control/fitting.py
index 36e01a8d..7b400855 100644
--- a/src/diffpy/pdfgui/control/fitting.py
+++ b/src/diffpy/pdfgui/control/fitting.py
@@ -410,9 +410,13 @@ def configure(self):
             #
             # Pair selection applies only to the current dataset,
             # therefore it has to be done here.
-            nstrucs = len(self.strucs)
-            for phaseidx, struc in zip(range(1, nstrucs + 1), self.strucs):
-                struc.applyPairSelection(self.server, phaseidx)
+            # TODO the following comment lines
+            # nstrucs = len(self.strucs)
+            # for phaseidx, struc in zip(range(1, nstrucs + 1), self.strucs):
+            #     print(phaseidx)
+            #     print(struc)
+            #     struc.applyCMIPairSelection()
+            #   struc.applyPairSelection(self.server, phaseidx)
 
 
         # turn on printout fithook in each refinement step
@@ -974,9 +978,9 @@ def cmiConstrain(self, key, var):
             self.cmirecipe.constrain(atoms[key_arg - 1].occupancy, var_name)
 
         # data parameters
-        if key_ascii_ref == 'qdamp':
+        if key_ref == 'qdamp':
             self.cmirecipe.constrain(self.cmipdfgen.qdamp, var_name)
-        if key_ascii_ref == 'qbroad':
+        if key_ref == 'qbroad':
             self.cmirecipe.constrain(self.cmipdfgen.qbroad, var_name)
         # TODO how to deal with `dscale`. cmipdfgen don't have `dscale` parameter.