=== added file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessEngine.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessEngine.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessEngine.java 2012-04-06 20:02:26 +0000 @@ -0,0 +1,46 @@ +package org.hisp.dhis.completeness; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Collection; + +import org.hisp.dhis.period.RelativePeriods; + +/** + * @author Lars Helge Overland + */ +public interface DataSetCompletenessEngine +{ + final String ID = DataSetCompletenessEngine.class.getName(); + + void exportDataSetCompleteness( Collection dataSetIds, RelativePeriods relatives, + Collection organisationUnitIds ); + + void exportDataSetCompleteness( Collection dataSetIds, Collection periodIds, + Collection organisationUnitIds ); +} === modified file 'dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessService.java' --- dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessService.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-api/src/main/java/org/hisp/dhis/completeness/DataSetCompletenessService.java 2012-04-06 20:02:26 +0000 @@ -28,8 +28,11 @@ */ import java.util.Collection; +import java.util.concurrent.Future; -import org.hisp.dhis.period.RelativePeriods; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.period.Period; /** * @author Lars Helge Overland @@ -39,23 +42,8 @@ { String ID = DataSetCompletenessService.class.getName(); - /** - * - * @param dataSetIds - * @param relatives - * @param organisationUnitIds - */ - void exportDataSetCompleteness( Collection dataSetIds, RelativePeriods relatives, - Collection organisationUnitIds ); - - /** - * - * @param dataSetIds - * @param periodIds - * @param organisationUnitIds - */ - void exportDataSetCompleteness( Collection dataSetIds, Collection periodIds, - Collection organisationUnitIds ); + Future exportDataSetCompleteness( Collection dataSets, Collection periods, + Collection units, int days ); /** * Returns a Collection of DataSetCompletenessResults. The === added directory 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/engine' === added file 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/engine/DefaultDataSetCompletenessEngine.java' --- dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/engine/DefaultDataSetCompletenessEngine.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/engine/DefaultDataSetCompletenessEngine.java 2012-04-06 20:02:26 +0000 @@ -0,0 +1,170 @@ +package org.hisp.dhis.completeness.engine; + +/* + * Copyright (c) 2004-2012, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.setting.SystemSettingManager.DEFAULT_COMPLETENESS_OFFSET; +import static org.hisp.dhis.setting.SystemSettingManager.KEY_COMPLETENESS_OFFSET; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.hisp.dhis.completeness.DataSetCompletenessEngine; +import org.hisp.dhis.completeness.DataSetCompletenessService; +import org.hisp.dhis.completeness.DataSetCompletenessStore; +import org.hisp.dhis.dataset.DataSet; +import org.hisp.dhis.dataset.DataSetService; +import org.hisp.dhis.organisationunit.OrganisationUnit; +import org.hisp.dhis.organisationunit.OrganisationUnitService; +import org.hisp.dhis.period.Period; +import org.hisp.dhis.period.PeriodService; +import org.hisp.dhis.period.RelativePeriods; +import org.hisp.dhis.setting.SystemSettingManager; +import org.hisp.dhis.system.util.ConcurrentUtils; +import org.hisp.dhis.system.util.ConversionUtils; +import org.hisp.dhis.system.util.PaginatedList; +import org.hisp.dhis.system.util.SystemUtils; +import org.springframework.transaction.annotation.Transactional; + +/** + * @author Lars Helge Overland + */ +public class DefaultDataSetCompletenessEngine + implements DataSetCompletenessEngine +{ + private static final Log log = LogFactory.getLog( DefaultDataSetCompletenessEngine.class ); + + // ------------------------------------------------------------------------- + // Dependencies + // ------------------------------------------------------------------------- + + private DataSetCompletenessService completenessService; + + public void setCompletenessService( DataSetCompletenessService completenessService ) + { + this.completenessService = completenessService; + } + + private DataSetCompletenessStore completenessStore; + + public void setCompletenessStore( DataSetCompletenessStore completenessStore ) + { + this.completenessStore = completenessStore; + } + + private OrganisationUnitService organisationUnitService; + + public void setOrganisationUnitService( OrganisationUnitService organisationUnitService ) + { + this.organisationUnitService = organisationUnitService; + } + + private PeriodService periodService; + + public void setPeriodService( PeriodService periodService ) + { + this.periodService = periodService; + } + + private DataSetService dataSetService; + + public void setDataSetService( DataSetService dataSetService ) + { + this.dataSetService = dataSetService; + } + + private SystemSettingManager systemSettingManager; + + public void setSystemSettingManager( SystemSettingManager systemSettingManager ) + { + this.systemSettingManager = systemSettingManager; + } + + // ------------------------------------------------------------------------- + // DataSetCompletenessEngine implementation + // ------------------------------------------------------------------------- + + @Transactional + public void exportDataSetCompleteness( Collection dataSetIds, RelativePeriods relatives, + Collection organisationUnitIds ) + { + if ( relatives != null ) + { + Collection periodIds = ConversionUtils.getIdentifiers( Period.class, + periodService.reloadPeriods( relatives.getRelativePeriods() ) ); + + exportDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); + } + } + + @Transactional + public void exportDataSetCompleteness( Collection dataSetIds, Collection periodIds, + Collection organisationUnitIds ) + { + final int cpuCores = SystemUtils.getCpuCores(); + + log.info( "Data completeness export process started" ); + + completenessStore.dropIndex(); + + log.info( "Dropped potential index" ); + + int days = (Integer) systemSettingManager.getSystemSetting( KEY_COMPLETENESS_OFFSET, + DEFAULT_COMPLETENESS_OFFSET ); + + completenessStore.deleteDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); + + log.info( "Deleted existing completeness data" ); + + Collection periods = periodService.getPeriods( periodIds ); + Collection organisationUnits = organisationUnitService.getOrganisationUnits( organisationUnitIds ); + Collection dataSets = dataSetService.getDataSets( dataSetIds ); + + dataSets = completenessStore.getDataSetsWithRegistrations( dataSets ); + + List> organisationUnitPages = new PaginatedList( organisationUnits ).setNumberOfPages( cpuCores ).getPages(); + + List> futures = new ArrayList>(); + + for ( List organisationUnitPage : organisationUnitPages ) + { + futures.add( completenessService.exportDataSetCompleteness( dataSets, periods, organisationUnitPage, days ) ); + } + + ConcurrentUtils.waitForCompletion( futures ); + + completenessStore.createIndex(); + + log.info( "Created index" ); + + log.info( "Completeness export process done" ); + } +} === modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/impl/AbstractDataSetCompletenessService.java' --- dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/impl/AbstractDataSetCompletenessService.java 2012-04-06 17:29:32 +0000 +++ dhis-2/dhis-services/dhis-service-reporting/src/main/java/org/hisp/dhis/completeness/impl/AbstractDataSetCompletenessService.java 2012-04-06 20:02:26 +0000 @@ -33,6 +33,8 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; +import java.util.concurrent.Future; import org.amplecode.quick.BatchHandler; import org.amplecode.quick.BatchHandlerFactory; @@ -52,16 +54,15 @@ import org.hisp.dhis.organisationunit.OrganisationUnitService; import org.hisp.dhis.period.Period; import org.hisp.dhis.period.PeriodService; -import org.hisp.dhis.period.RelativePeriods; import org.hisp.dhis.setting.SystemSettingManager; import org.hisp.dhis.system.util.ConversionUtils; +import org.springframework.scheduling.annotation.Async; import org.springframework.transaction.annotation.Transactional; /** * @author Lars Helge Overland * @version $Id$ */ -@Transactional public abstract class AbstractDataSetCompletenessService implements DataSetCompletenessService { @@ -135,51 +136,21 @@ // Abstract methods // ------------------------------------------------------------------------- - public abstract int getRegistrations( DataSet dataSet, Collection relevantSources, Collection periods ); + protected abstract int getRegistrations( DataSet dataSet, Collection relevantSources, Collection periods ); - public abstract int getRegistrationsOnTime( DataSet dataSet, Collection relevantSources, Collection periods, + protected abstract int getRegistrationsOnTime( DataSet dataSet, Collection relevantSources, Collection periods, int completenessOffset ); - public abstract int getSources( DataSet dataSet, Collection relevantSources, Period period ); + protected abstract int getSources( DataSet dataSet, Collection relevantSources, Period period ); // ------------------------------------------------------------------------- // DataSetCompleteness // ------------------------------------------------------------------------- - public void exportDataSetCompleteness( Collection dataSetIds, RelativePeriods relatives, - Collection organisationUnitIds ) - { - if ( relatives != null ) - { - Collection periodIds = ConversionUtils.getIdentifiers( Period.class, - periodService.reloadPeriods( relatives.getRelativePeriods() ) ); - - exportDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); - } - } - - public void exportDataSetCompleteness( Collection dataSetIds, Collection periodIds, - Collection organisationUnitIds ) - { - log.info( "Data completeness export process started" ); - - completenessStore.dropIndex(); - - log.info( "Dropped potential index" ); - - int days = (Integer) systemSettingManager.getSystemSetting( KEY_COMPLETENESS_OFFSET, - DEFAULT_COMPLETENESS_OFFSET ); - - completenessStore.deleteDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); - - log.info( "Deleted existing completeness data" ); - - Collection periods = periodService.getPeriods( periodIds ); - Collection units = organisationUnitService.getOrganisationUnits( organisationUnitIds ); - Collection dataSets = dataSetService.getDataSets( dataSetIds ); - - dataSets = completenessStore.getDataSetsWithRegistrations( dataSets ); - + @Async + public Future exportDataSetCompleteness( Collection dataSets, Collection periods, + Collection units, int days ) + { BatchHandler batchHandler = batchHandlerFactory .createBatchHandler( DataSetCompletenessResultBatchHandler.class ).init(); @@ -213,21 +184,18 @@ } } } - - log.info( "Exported completeness for data set: " + dataSet ); } batchHandler.flush(); aggregationCache.clearCache(); - completenessStore.createIndex(); - - log.info( "Created index" ); - - log.info( "Completeness export process done" ); + log.info( "Completeness export task done" ); + + return null; } + @Transactional public Collection getDataSetCompleteness( int periodId, int organisationUnitId ) { final Period period = periodService.getPeriod( periodId ); @@ -270,6 +238,7 @@ return results; } + @Transactional public Collection getDataSetCompleteness( int periodId, Collection organisationUnitIds, int dataSetId ) { @@ -303,7 +272,8 @@ return results; } - + + @Transactional public void deleteDataSetCompleteness() { completenessStore.deleteDataSetCompleteness(); @@ -313,11 +283,13 @@ // Index // ------------------------------------------------------------------------- + @Transactional public void createIndex() { completenessStore.createIndex(); } + @Transactional public void dropIndex() { completenessStore.dropIndex(); @@ -353,7 +325,7 @@ private Collection getRelevantSources( DataSet dataSet, Collection sources ) { Collection dataSetSources = ConversionUtils.getIdentifiers( OrganisationUnit.class, - dataSet.getSources() ); + new HashSet( dataSet.getSources() ) ); return CollectionUtils.intersection( dataSetSources, sources ); } === modified file 'dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2012-03-16 16:49:58 +0000 +++ dhis-2/dhis-services/dhis-service-reporting/src/main/resources/META-INF/dhis/beans.xml 2012-04-06 20:02:26 +0000 @@ -98,6 +98,15 @@ + + + + + + + + + @@ -256,7 +265,7 @@ - + === modified file 'dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/completeness/DataSetCompletenessServiceExportTest.java' --- dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/completeness/DataSetCompletenessServiceExportTest.java 2012-03-15 16:04:20 +0000 +++ dhis-2/dhis-services/dhis-service-reporting/src/test/java/org/hisp/dhis/completeness/DataSetCompletenessServiceExportTest.java 2012-04-06 20:02:26 +0000 @@ -45,7 +45,6 @@ import org.hisp.dhis.period.PeriodService; import org.hisp.dhis.period.PeriodType; import org.hisp.dhis.period.QuarterlyPeriodType; -import org.hisp.dhis.reporttable.ReportTable; import org.junit.Test; /** @@ -55,10 +54,10 @@ public class DataSetCompletenessServiceExportTest extends DhisTest { + private DataSetCompletenessEngine completenessEngine; + private DataSetCompletenessStore completenessStore; - private DataSetCompletenessService completenessService; - private CompleteDataSetRegistrationService registrationService; private PeriodType monthly; @@ -82,10 +81,10 @@ @Override public void setUpTest() { + completenessEngine = (DataSetCompletenessEngine) getBean( DataSetCompletenessEngine.ID ); + completenessStore = (DataSetCompletenessStore) getBean( DataSetCompletenessStore.ID ); - completenessService = (DataSetCompletenessService) getBean( "registrationDataCompletenessService" ); - periodService = (PeriodService) getBean( PeriodService.ID ); organisationUnitService = (OrganisationUnitService) getBean( OrganisationUnitService.ID ); @@ -161,7 +160,7 @@ registrationService.saveCompleteDataSetRegistration( new CompleteDataSetRegistration( dataSetA, periodC, unitA, null, "" ) ); registrationService.saveCompleteDataSetRegistration( new CompleteDataSetRegistration( dataSetA, periodC, unitC, null, "" ) ); - completenessService.exportDataSetCompleteness( getIdentifiers( DataSet.class, dataSets ), + completenessEngine.exportDataSetCompleteness( getIdentifiers( DataSet.class, dataSets ), getIdentifiers( Period.class, periods ), getIdentifiers( OrganisationUnit.class, units ) ); assertEquals( 100.0, completenessStore.getPercentage( dataSetA.getId(), periodA.getId(), unitB.getId() ) ); === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java 2012-02-12 20:32:14 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/scheduling/DataMartTask.java 2012-04-06 20:02:26 +0000 @@ -38,7 +38,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.hisp.dhis.completeness.DataSetCompletenessService; +import org.hisp.dhis.completeness.DataSetCompletenessEngine; import org.hisp.dhis.dataelement.DataElement; import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.datamart.DataMartEngine; @@ -70,7 +70,7 @@ private DataMartEngine dataMartEngine; - private DataSetCompletenessService completenessService; + private DataSetCompletenessEngine completenessEngine; private DataElementService dataElementService; @@ -119,13 +119,13 @@ { } - public DataMartTask( DataMartEngine dataMartEngine, DataSetCompletenessService completenessService, + public DataMartTask( DataMartEngine dataMartEngine, DataSetCompletenessEngine completenessEngine, DataElementService dataElementService, IndicatorService indicatorService, PeriodService periodService, OrganisationUnitService organisationUnitService, OrganisationUnitGroupService organisationUnitGroupService, DataSetService dataSetService, SystemSettingManager systemSettingManager ) { this.dataMartEngine = dataMartEngine; - this.completenessService = completenessService; + this.completenessEngine = completenessEngine; this.dataElementService = dataElementService; this.indicatorService = indicatorService; this.periodService = periodService; @@ -158,7 +158,7 @@ Collection periodIds = ConversionUtils.getIdentifiers( Period.class, periodService.reloadPeriods( periods ) ); dataMartEngine.export( dataElementIds, indicatorIds, periodIds, organisationUnitIds, organisationUnitGroupIds ); - completenessService.exportDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); + completenessEngine.exportDataSetCompleteness( dataSetIds, periodIds, organisationUnitIds ); } // ------------------------------------------------------------------------- === modified file 'dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ConcurrentUtils.java' --- dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ConcurrentUtils.java 2011-12-26 10:07:59 +0000 +++ dhis-2/dhis-support/dhis-support-system/src/main/java/org/hisp/dhis/system/util/ConcurrentUtils.java 2012-04-06 20:02:26 +0000 @@ -28,6 +28,7 @@ */ import java.util.Collection; +import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; /** @@ -48,9 +49,13 @@ { future.get(); } - catch ( Exception ex ) - { - throw new RuntimeException( ex ); + catch ( ExecutionException ex ) + { + throw new RuntimeException( "Exception during execution", ex ); + } + catch ( InterruptedException ex ) + { + throw new RuntimeException( "Thread interrupted", ex ); } } }