-
Notifications
You must be signed in to change notification settings - Fork 175
/
low_order_moms_streaming.py
74 lines (61 loc) · 2.51 KB
/
low_order_moms_streaming.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# ==============================================================================
# Copyright 2014 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# daal4py low order moments example for streaming on shared memory systems
from pathlib import Path
from readcsv import pd_read_csv
import daal4py as d4p
def main(readcsv=pd_read_csv, *args, **kwargs):
# read data from file
data_path = Path(__file__).parent / "data" / "batch"
file = data_path / "covcormoments_dense.csv"
# Configure a low order moments object for streaming
algo = d4p.low_order_moments(streaming=True)
chunk_size = 55
lines_read = 0
# read and feed chunk by chunk
while True:
# Read data in chunks
try:
data = readcsv(
file, usecols=range(10), skip_header=lines_read, max_rows=chunk_size
)
except StopIteration as e:
if lines_read > 0:
break
else:
raise ValueError("No training data was read - empty input file?") from e
# Now feed chunk
algo.compute(data)
lines_read += data.shape[0]
# All files are done, now finalize the computation
result = algo.finalize()
# result provides minimum, maximum, sum, sumSquares, sumSquaresCentered,
# mean, secondOrderRawMoment, variance, standardDeviation, variation
return result
if __name__ == "__main__":
res = main()
# print results
print("\nMinimum:\n", res.minimum)
print("\nMaximum:\n", res.maximum)
print("\nSum:\n", res.sum)
print("\nSum of squares:\n", res.sumSquares)
print("\nSum of squared difference from the means:\n", res.sumSquaresCentered)
print("\nMean:\n", res.mean)
print("\nSecond order raw moment:\n", res.secondOrderRawMoment)
print("\nVariance:\n", res.variance)
print("\nStandard deviation:\n", res.standardDeviation)
print("\nVariation:\n", res.variation)
print("All looks good!")