Description

This test ensures the correctness of the Connect agent CSVWriter.

Contributor: Michael Bussieck, March 2022

Small Model of Type : GAMS

Category : GAMS Test library

Main file : cacsvw.gms

$title 'Test Connect agent CSVWriter' (CACSVW,SEQ=895)
$ontext
This test ensures the correctness of the Connect agent CSVWriter.

Contributor: Michael Bussieck, March 2022
$offtext


* On the major platforms (Windows, Linux, Mac), GMSPYTHONLIB gets automatically set 
* to use the internal Python installation in sysdir/GMSPython.
$if not setEnv GMSPYTHONLIB $abort.noError Embedded code Python not ready to be used
$log --- Using Python library %sysEnv.GMSPYTHONLIB%

$onechoV > t.gms
Set i(*) canning plants /
'seattle',
'san-diego' /;

Set j(*) markets /
'new-york',
'chicago',
'topeka' /;

Parameter a(i) capacity of plant i in cases /
'seattle' 350,
'san-diego' 600 /;

Parameter b(j) demand at market j in cases /
'new-york' 325,
'chicago' 300,
'topeka' 275 /;

Parameter d(i,j) distance in thousands of miles /
'seattle'.'new-york' 2.5,
'seattle'.'chicago' 1.7,
'seattle'.'topeka' 1.8,
'san-diego'.'new-york' 2.5,
'san-diego'.'chicago' 1.8,
'san-diego'.'topeka' 1.4 /;

Scalar f freight in dollars per case per thousand miles / 90 /;

Parameter c(i,j) transport cost in thousands of dollars per case /
'seattle'.'new-york' 0.225,
'seattle'.'chicago' 0.153,
'seattle'.'topeka' 0.162,
'san-diego'.'new-york' 0.225,
'san-diego'.'chicago' 0.162,
'san-diego'.'topeka' 0.126 /;

positive Variable x(i,j) shipment quantities in cases /
'seattle'.'new-york'.L 50,
'seattle'.'chicago'.L 300,
'seattle'.'topeka'.M 0.036,
'san-diego'.'new-york'.L 275,
'san-diego'.'chicago'.M 0.009,
'san-diego'.'topeka'.L 275 /;

free     Variable z total transportation costs in thousands of dollars /L 153.675 /;

Equation cost define objective function /M 1, LO 0, UP 0 /;

Equation supply(i) observe supply limit at plant i /
'seattle'.L 350,
'seattle'.M Eps,
'seattle'.LO -Inf,
'seattle'.UP 350,
'san-diego'.L 550,
'san-diego'.LO -Inf,
'san-diego'.UP 600 /;

Equation demand(j) satisfy demand at market j /
'new-york'.L 325,
'new-york'.M 0.225,
'new-york'.LO 325,
'new-york'.UP +Inf,
'chicago'.L 300,
'chicago'.M 0.153,
'chicago'.LO 300,
'chicago'.UP +Inf,
'topeka'.L 275,
'topeka'.M 0.126,
'topeka'.LO 275,
'topeka'.UP +Inf /;

$offEmpty
$offEcho
$call.checkErrorLevel gams t.gms lo=%gams.lo% gdx t.gdx

$onEcho >  d_ref.csv
seattle,new-york,2.5
seattle,chicago,1.7
seattle,topeka,1.8
san-diego,new-york,2.5
san-diego,chicago,1.8
san-diego,topeka,1.4
$offEcho
$onEcho >  demand_ref.csv
new-york,325.0,0.225,325.0,inf,1.0
chicago,300.0,0.153,300.0,inf,1.0
topeka,275.0,0.126,275.0,inf,1.0
$offEcho
$onEcho >  f_ref.csv
90.0
$offEcho
$onEcho >  j_ref.csv
new-york,
chicago,
topeka,
$offEcho
$onEcho >  x_ref.csv
seattle,new-york,50.0,0.0,0.0,inf,1.0
seattle,chicago,300.0,0.0,0.0,inf,1.0
seattle,topeka,0.0,0.036,0.0,inf,1.0
san-diego,new-york,275.0,0.0,0.0,inf,1.0
san-diego,chicago,0.0,0.009,0.0,inf,1.0
san-diego,topeka,275.0,0.0,0.0,inf,1.0
$offEcho
$onEcho >  x.l_ref.csv
seattle,new-york,50.0
seattle,chicago,300.0
seattle,topeka,0.0
san-diego,new-york,275.0
san-diego,chicago,0.0
san-diego,topeka,275.0
$offEcho
$onEcho >  x.lo_ref.csv
seattle,new-york,0.0
seattle,chicago,0.0
seattle,topeka,0.0
san-diego,new-york,0.0
san-diego,chicago,0.0
san-diego,topeka,0.0
$offEcho
$onEcho >  x.scale_ref.csv
seattle,new-york,1.0
seattle,chicago,1.0
seattle,topeka,1.0
san-diego,new-york,1.0
san-diego,chicago,1.0
san-diego,topeka,1.0
$offEcho
$onEcho >  x.up_ref.csv
seattle,new-york,inf
seattle,chicago,inf
seattle,topeka,inf
san-diego,new-york,inf
san-diego,chicago,inf
san-diego,topeka,inf
$offEcho
$onEcho >  z_ref.csv
153.675,0.0,-inf,inf,1.0
$offEcho
$onEcho >  d_header_ref.csv
Dim1,Dim2,Val
seattle,new-york,2.5
seattle,chicago,1.7
seattle,topeka,1.8
san-diego,new-york,2.5
san-diego,chicago,1.8
san-diego,topeka,1.4
$offEcho
$onEcho >  demand_header_ref.csv
Dim1,Level,Marginal,Lower,Upper,Scale
new-york,325.0,0.225,325.0,inf,1.0
chicago,300.0,0.153,300.0,inf,1.0
topeka,275.0,0.126,275.0,inf,1.0
$offEcho
$onEcho >  f_header_ref.csv
Val
90.0
$offEcho
$onEcho >  j_header_ref.csv
Dim1,Text
new-york,
chicago,
topeka,
$offEcho
$onEcho >  x.l_header_ref.csv
Dim1,Dim2,Level
seattle,new-york,50.0
seattle,chicago,300.0
seattle,topeka,0.0
san-diego,new-york,275.0
san-diego,chicago,0.0
san-diego,topeka,275.0
$offEcho
$onEcho >  x.lo_header_ref.csv
Dim1,Dim2,Lower
seattle,new-york,0.0
seattle,chicago,0.0
seattle,topeka,0.0
san-diego,new-york,0.0
san-diego,chicago,0.0
san-diego,topeka,0.0
$offEcho
$onEcho >  x.scale_header_ref.csv
Dim1,Dim2,Scale
seattle,new-york,1.0
seattle,chicago,1.0
seattle,topeka,1.0
san-diego,new-york,1.0
san-diego,chicago,1.0
san-diego,topeka,1.0
$offEcho
$onEcho >  x.up_header_ref.csv
Dim1,Dim2,Upper
seattle,new-york,inf
seattle,chicago,inf
seattle,topeka,inf
san-diego,new-york,inf
san-diego,chicago,inf
san-diego,topeka,inf
$offEcho
$onEcho >  x_header_ref.csv
Dim1,Dim2,Level,Marginal,Lower,Upper,Scale
seattle,new-york,50.0,0.0,0.0,inf,1.0
seattle,chicago,300.0,0.0,0.0,inf,1.0
seattle,topeka,0.0,0.036,0.0,inf,1.0
san-diego,new-york,275.0,0.0,0.0,inf,1.0
san-diego,chicago,0.0,0.009,0.0,inf,1.0
san-diego,topeka,275.0,0.0,0.0,inf,1.0
$offEcho
$onEcho >  z_header_ref.csv
Level,Marginal,Lower,Upper,Scale
153.675,0.0,-inf,inf,1.0
$offEcho
$onEcho >  d1_ref.csv
Dim1;new-york;chicago;topeka
seattle;2.5;1.7;1.8
san-diego;2.5;1.8;1.4
$offEcho
$onEcho >  d2_ref.csv
Dim1;new-york;chicago;topeka
seattle;2,5;1,7;1,8
san-diego;2,5;1,8;1,4
$offEcho
$onEcho >  d3_ref.csv
Dim1,new-york,chicago,topeka
seattle,2.5,1.7,1.8
san-diego,2.5,1.8,1.4
$offEcho
$onEcho >  d4_ref.csv
"Dim1","new-york","chicago","topeka"
"seattle","2.5","1.7","1.8"
"san-diego","2.5","1.8","1.4"
$offEcho
$onEcho >  d5_ref.csv
"Dim1","new-york","chicago","topeka"
"seattle",2.5,1.7,1.8
"san-diego",2.5,1.8,1.4
$offEcho
$onEcho >  d6_ref.csv
seattle,2.5,1.7,1.8
san-diego,2.5,1.8,1.4
$offEcho
$onEcho >  x1_ref.csv
i,j,Upper
seattle,new-york,inf
seattle,chicago,inf
seattle,topeka,inf
san-diego,new-york,inf
san-diego,chicago,inf
san-diego,topeka,inf
$offEcho
$onEcho > x2_ref.csv
Dim1,new-york,chicago,topeka
seattle,50.0,300.0,0.0
san-diego,275.0,0.0,275.0
$offEcho

set sym / j, d, f, demand, x, 'x.l', 'x.lo', 'x.up', 'x.scale', z /;
loop(sym,
   put_utility 'ecArguments' / '--SYM=' sym.tl:0;
   EmbeddedCode Connect:
   - GDXReader:
       file: t.gdx
       readAll: True
   - CSVWriter:
       file: %SYM%.csv
       name: %SYM%
       header: False
   endEmbeddedCode
   put_utility 'shell.checkErrorLevel' / 'diff -q ' sym.tl:0 '.csv ' sym.tl:0 '_ref.csv' ' > %system.nullFile%';
);   
loop(sym,
   put_utility 'ecArguments' / '--SYM=' sym.tl:0;
   EmbeddedCode Connect:
   - GDXReader:
       file: t.gdx
       readAll: True
   - CSVWriter:
       file: %SYM%_header.csv
       name: %SYM%
       header: True
   endEmbeddedCode
   put_utility 'shell.checkErrorLevel' / 'diff -q ' sym.tl:0 '_header.csv ' sym.tl:0 '_header_ref.csv' ' > %system.nullFile%';
);   

put_utility 'ecArguments' / '';
EmbeddedCode Connect:
- GDXReader:
    file: t.gdx
    readAll: True
- CSVWriter:
    file: d1.csv
    name: d
    header: True
    unstack: True
    fieldSeparator: ;
- CSVWriter:
    file: d2.csv
    name: d
    header: True
    unstack: True
    fieldSeparator: ;
    decimalSeparator: ","
- CSVWriter:
    file: d3.csv
    name: d
    header: True
    unstack: True
    quoting: 0
- CSVWriter:
    file: d4.csv
    name: d
    header: True
    unstack: True
    quoting: 1
- CSVWriter:
    file: d5.csv
    name: d
    header: True
    unstack: True
    quoting: 2
- CSVWriter:
    file: d6.csv
    name: d
    header: False
    unstack: True
- CSVWriter:
    file: x1.csv
    name: x.up
    header: True
    setHeader: i,j,Upper
- CSVWriter:
    file: x2.csv
    name: x.l
    header: True
    unstack: True
endEmbeddedCode
put_utility 'shell.checkErrorLevel' / 'diff -q d1.csv d1_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q d2.csv d2_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q d3.csv d3_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q d4.csv d4_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q d5.csv d5_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q d6.csv d6_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q x1.csv x1_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q x2.csv x2_ref.csv' ' > %system.nullFile%';

$onEcho > sv1_ref.csv
Dim1,Val
1,-0.0
2,-inf
3,inf
4,
5,
$offEcho
$onEcho > sv2_ref.csv
Dim1,Val
1,-0.0
2,-inf
3,inf
4,NA
5,NA
$offEcho
$onundf
parameter sv / 1 eps, 2 -inf, 3 +inf, 4 na, 5 undf /;
EmbeddedCode Connect:
- GAMSReader:
    readAll: True
- CSVWriter:
    file: sv1.csv
    name: sv
    header: True
- CSVWriter:
    file: sv2.csv
    name: sv
    header: True
    toCSVArguments: { na_rep: 'NA'}
endEmbeddedCode
put_utility 'shell.checkErrorLevel' / 'diff -q sv1.csv sv1_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q sv2.csv sv2_ref.csv' ' > %system.nullFile%';

$onEcho > s1_ref.csv
Dim1,Text
1,One
2,Two
3,Three
$offEcho
$onEcho > s2_ref.csv
Dim1
1
2
3
$offEcho
set s / 1 One, 2 Two, 3 Three /;
EmbeddedCode Connect:
- GAMSReader:
    readAll: True
- CSVWriter:
    file: s1.csv
    name: s
    header: True
- CSVWriter:
    file: s2.csv
    name: s
    header: True
    skipElementText: True
endEmbeddedCode
put_utility 'shell.checkErrorLevel' / 'diff -q s1.csv s1_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q s2.csv s2_ref.csv' ' > %system.nullFile%';

$onEcho > t1_ref.csv
Dim1,Dim2,Text
1,1,One
2,2,Two
3,3,Three
$offEcho
$onEcho > t2_ref.csv
Dim1,Dim2
1,1
2,2
3,3
$offEcho
$onEcho > t3_ref.csv
Dim1,1,2,3
1,One,,
2,,Two,
3,,,Three
$offEcho
$onEcho > t4_ref.csv
Dim1,1,2,3
1,Y,,
2,,Y,
3,,,Y
$offEcho
$onEcho > t5_ref.csv
1,Y,,
2,,Y,
3,,,Y
$offEcho
set t / 1.1 One, 2.2 Two, 3.3 Three /;
EmbeddedCode Connect:
- GAMSReader:
    readAll: True
- CSVWriter:
    file: t1.csv
    name: t
    header: True
- CSVWriter:
    file: t2.csv
    name: t
    header: True
    skipElementText: True
- CSVWriter:
    file: t3.csv
    name: t
    header: True
    unstack: True
- CSVWriter:
    file: t4.csv
    name: t
    header: True
    unstack: True
    skipElementText: True
- CSVWriter:
    file: t5.csv
    name: t
    header: True
    unstack: True
    skipElementText: True
    toCSVArguments: { mode: w } # overwrite mode=a from setHeader
    setHeader: 'i,1,2,3'
endEmbeddedCode
put_utility 'shell.checkErrorLevel' / 'diff -q t1.csv t1_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q t2.csv t2_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q t3.csv t3_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q t4.csv t4_ref.csv' ' > %system.nullFile%';
put_utility 'shell.checkErrorLevel' / 'diff -q t5.csv t5_ref.csv' ' > %system.nullFile%';

* we expect the correct uel order to be written: j1, j2, j3, j4
$onEcho > p2_ref.csv
Dim1,j1,j2,j3,j4
i1,,,2.0,1.0
i2,4.0,3.0,,
$offEcho
EmbeddedCode Connect:
- PythonCode:
    code: |
      universe = connect.db.add_set('universe', 1)
      universe.add_record('i1')
      universe.add_record('i2')
      universe.add_record('j1')
      universe.add_record('j2')
      universe.add_record('j3')
      universe.add_record('j4')

      p2 = connect.db.add_parameter('p2', 2)
      p2.add_record(('i1', 'j4')).value = 1
      p2.add_record(('i1', 'j3')).value = 2
      p2.add_record(('i2', 'j2')).value = 3
      p2.add_record(('i2', 'j1')).value = 4
- CSVWriter:
    file: p2_order.csv
    name: p2
    unstack: True
endEmbeddedCode
put_utility 'shell.checkErrorLevel' / 'diff -q p2_order.csv p2_ref.csv' ' > %system.nullFile%';