Modifcations for searching the targeted eigenpairs one by one

Author: CYChenFudan

Changes_cychen.txt

@@ -14,3 +14,4 @@ monitor. The output messages are very useful for extremely large-scaled calculat
 iteration calculations. Or, error results could be obtained in some cases (in
 O-like isoelectronic sequence?). 
 09. dvdson--mpi
+10. Modifcations for searching the targeted eigenpairs one by one

src/appl/rci90_mpi/maneigmpi.f90

@@ -132,8 +132,12 @@ SUBROUTINE MANEIG(IATJPO, IASPAR, NELMNT_a)
       CHARACTER(LEN=8) :: CNUM
       REAL(DOUBLE), DIMENSION(:), pointer :: w, z, work, diag
       INTEGER, DIMENSION(:), pointer :: iwork, ifail, jwork
-!-----------------------------------------------
-!
+
+!-----------------------------------------------------------------------
+! CYC Modifications for step by step diagonalization
+      INTEGER IRESTART_GDVD, ISTEP, NSTEP, IITMP, NMV0
+      INTEGER TIME0, TIME1, TIME2
+! CYC
 !-----------------------------------------------------------------------
       ABSTOL = 2*DLAMCH('S')
       IF (MYID == 0) WRITE (6, *) 'Calling maneig...'
@@ -494,7 +498,7 @@ SUBROUTINE MANEIG(IATJPO, IASPAR, NELMNT_a)
               print *, 'Returned from iniestsd '
               if (ncf.gt. IOLPCK) then
                 print *, 'Calling GDVD'
-                CALL GDVD (SPODMV,NCF,LIM,DIAG,ILOW,IHIGH,            &
+                CALL GDVD (SPODMV,IRESTART_GDVD,NCF,LIM,DIAG,ILOW,IHIGH,            &
                   JWORK,NIV,MBLOCK,CRITE,CRITC, CRITR,ORTHO,MAXITR,   &
                   WORK,LWORK,IWORK,LIWORK,HIEND,NLOOPS,               &
                   NMV,IERR)
@@ -505,10 +509,70 @@ SUBROUTINE MANEIG(IATJPO, IASPAR, NELMNT_a)
                IF (myid .EQ. 0) print *, ' Sparse - Memory, iniestmpi'
                CALL iniestmpi (IOLPCK, NCF,NIV,WORK,EMT,IENDC,IROW)
                if(ncf.gt. IOLPCK) then
-                 CALL GDVD (SPICMVmpi,NCF,LIM,DIAG,ILOW,IHIGH,        &
+!-----------------------------------------------------------------------
+! CYC: Try to diagonalize step by step
+                IRESTART_GDVD = 0
+                TIME0 = MPI_WTIME()
+! More tests should be performed for istep, it could be set as 1
+                ISTEP = 5 
+                NMV = 0 
+                IF (NCF.GT.2.0E5 .AND. NVEX.GT.ISTEP) THEN
+                 NSTEP = NVEX / ISTEP
+                 IF (MOD(NVEX,ISTEP).LE.ISTEP/2) NSTEP = NSTEP - 1
+                 DO IITMP = 1, NSTEP
+! NIV could be changed to nvecmx+1 in dvdson:initdvd procedure
+                  NIV = NVEX
+                  ILOW = 1
+                  IHIGH = IITMP * ISTEP
+                  IF (IITMP .EQ. 1 ) THEN
+                    IRESTART_GDVD = 0
+                  ELSE
+                    IRESTART_GDVD = 1
+                  ENDIF
+                  NMV0 = NMV
+                  TIME1 = MPI_WTIME()
+
+                  CALL GDVD (SPICMVmpi,IRESTART_GDVD,NCF,LIM,DIAG,ILOW,IHIGH, &
                   JWORK,NIV,MBLOCK,CRITE,CRITC, CRITR,ORTHO,MAXITR,   &
                   WORK,LWORK,IWORK,LIWORK,HIEND,NLOOPS,               &
                   NMV,IERR)
+
+                  TIME2 = MPI_WTIME()
+                  IF (MYID .EQ. 0) THEN
+                    WRITE (*, *)' IITMP= ', IITMP,  &
+                                ' Time (s)=', TIME2 - TIME1
+                    WRITE (*,*) ' ', NMV - NMV0, &
+                      'Matrix-vector multiplies for this loop.'  
+                    WRITE (*,*) ' ', NMV,  &
+                      ' Total matrix-vector multiplies by now.'
+                  ENDIF
+                 ENDDO
+                ENDIF
+
+! Search the remaining (NVEX - ISTEP * NSTEP) EIGENPAIRS
+                NIV = NVEX
+                ILOW = 1
+                IHIGH = NVEX
+                NMV0 = NMV  
+                TIME1 = MPI_WTIME()
+
+                CALL GDVD(SPICMVmpi,IRESTART_GDVD,NCF,LIM,DIAG,ILOW,IHIGH, &
+                JWORK,NIV,MBLOCK,CRITE,CRITC, CRITR,ORTHO,MAXITR,   &
+                WORK,LWORK,IWORK,LIWORK,HIEND,NLOOPS,               &
+                NMV,IERR)
+
+                TIME2 = MPI_WTIME()
+                IF (MYID .EQ. 0) THEN
+                  WRITE (*, *)' IITMP= ', IITMP,  &
+                              ' Time (s)=', TIME2 - TIME1 
+                  WRITE (*,*) ' ', NMV - NMV0, &
+                    'Matrix-vector multiplies for this loop.'  
+                  WRITE (*,*) ' ', NMV,  &
+                    ' Total matrix-vector multiplies by now.'
+                  WRITE (*, *)' Total time (m)=', (TIME2 - TIME0) / 60.0
+                ENDIF
+! CYC
+!-----------------------------------------------------------------------
                end if
 
                CALL DALLOC (EMT, 'EMT', 'MANEIG')
@@ -520,7 +584,7 @@ SUBROUTINE MANEIG(IATJPO, IASPAR, NELMNT_a)
                IF (myid .EQ. 0) print *, ' Dense - Memory, iniestdm'
               CALL INIESTDM (IOLPCK,NCF,NIV,WORK,EMT)
               if (ncf.gt. IOLPCK) then
-                CALL GDVD (DNICMV,NCF,LIM,DIAG,ILOW,IHIGH,            &
+                CALL GDVD (DNICMV,IRESTART_GDVD,NCF,LIM,DIAG,ILOW,IHIGH,            &
                     JWORK,NIV,MBLOCK,CRITE,CRITC, CRITR,ORTHO,MAXITR, &
                     WORK,LWORK,IWORK,LIWORK,HIEND,NLOOPS,             &
                     NMV,IERR)

src/lib/libdvd90/dvdson.f90

@@ -362,7 +362,11 @@ SUBROUTINE DVDSON(IRC, IREV, N, LIM, NOC, ILOW, IHIGH, ISELEC, NIV, &
 
       IF (IERR /= 0) RETURN
 
-      NUME = IHIGH
+!set NUME=NIV. In step-by-step diagonalization, IHIGH is changed in
+!different calling dvdson
+      !NUME=IHIGH
+      NUME = NIV
+
 !       ..Identify if few of the highest eigenpairs are wanted.
       IF (ILOW + IHIGH - 1 > N) THEN
          HIEND = .TRUE.
@@ -423,6 +427,7 @@ SUBROUTINE DVDSON(IRC, IREV, N, LIM, NOC, ILOW, IHIGH, ISELEC, NIV, &
 !
 
   100 CONTINUE
+! CYC: niv may be changed to nume+1 in initdvd
       CALL INITDVD (IRC, IREV, N, NOC, NIV, NUME + 1, LIM, HIEND, WORK(ISCRA1)&
          , WORK(IORTHO), WORK(IBASIS), WORK(IAB), WORK(IS))
 !       ----------------------------------------------------------------

src/lib/libdvd90/gdvd.f90

@@ -1,7 +1,7 @@
 
 !***************************************************************************
 
-      SUBROUTINE GDVD(OP, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK, CRITE&
+      SUBROUTINE GDVD(OP, IRESTART_GDVD, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK, CRITE&
          , CRITC, CRITR, ORTHO, MAXITER, WORK, IWRSZ, IWORK, IIWSZ, HIEND, &
          NLOOPS, NMV, IERR)
 !      Written by M. Saparov
@@ -25,6 +25,7 @@ SUBROUTINE GDVD(OP, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK, CRITE&
 !-----------------------------------------------
 !   D u m m y   A r g u m e n t s
 !-----------------------------------------------
+      INTEGER  :: IRESTART_GDVD 
       INTEGER  :: N
       INTEGER  :: LIM
       INTEGER  :: ILOW
@@ -134,13 +135,27 @@ SUBROUTINE GDVD(OP, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK, CRITE&
          GO TO 99
 
 !*********
-      ELSE IF (IRC==2 .OR. IRC==3) THEN
+!-----------------------------------------------------------------------
+! CYC: Try to diagonalize step by step
+      !ELSE IF (IRC==2 .OR. IRC==3) THEN
+      ELSE IF (IRC==3) THEN
 !********* ..Matrix-vector multiply.
          CALL OP (N, NB, WORK(IW1), WORK(IW2))
          NMV = NMV + NB
-
          GO TO 99
+
+      ELSE IF (IRC==2) THEN
+       IF (IRESTART_GDVD==0) THEN
+         CALL OP (N, NB, WORK(IW1), WORK(IW2))
+         NMV = NMV + NB 
+       ELSE
+         CALL OP (N, 1, WORK(IW1), WORK(IW2))
+         NMV = NMV + 1
+       ENDIF
+       GO TO 99
       ENDIF
+! CYC
+!-----------------------------------------------------------------------
 ! * * * * End of Reverse Communication * * * * * * * * * * * * * * * * *
 
       RETURN

src/lib/libdvd90/gdvd_I.f90

@@ -3,11 +3,12 @@ MODULE gdvd_I
 !...Generated by Pacific-Sierra Research 77to90  4.3E  20:12:31   2/12/04
 !...Modified by Charlotte Froese Fischer
 !                     Gediminas Gaigalas  10/05/17
-      SUBROUTINE gdvd (OP, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK&
+      SUBROUTINE gdvd (OP, IRESTART_GDVD, N, LIM, DIAG, ILOW, IHIGH, ISELEC, NIV, MBLOCK&
          , CRITE, CRITC, CRITR, ORTHO, MAXITER, WORK, IWRSZ, IWORK, IIWSZ&
          , HIEND, NLOOPS, NMV, IERR)
       USE vast_kind_param,ONLY: DOUBLE
       EXTERNAL OP
+      integer, INTENT(IN) :: IRESTART_GDVD
       integer, INTENT(IN) :: N
       integer, INTENT(IN) :: LIM
       real(DOUBLE), DIMENSION(N), INTENT(INOUT) :: DIAG